Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/torch_profiler/rank0.1780383301809004862.pt.trace.json filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/run-3241szoo.wandb filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/torch_profiler/rank0.1780383283598487142.pt.trace.json filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/run-ixxjov67.wandb filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/torch_profiler/rank0.1780383278694422947.pt.trace.json filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/run-nn58xrpu.wandb filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/torch_profiler/rank0.1780391935713307947.pt.trace.json filter=lfs diff=lfs merge=lfs -text
+ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/run-iwxrpu9h.wandb filter=lfs diff=lfs merge=lfs -text

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+#SBATCH --job-name=14b_8node_288x512_pair1x49_next_eb32_lr2e5
+#SBATCH --nodes=8
+#SBATCH --gpus=32
+#SBATCH --gpus-per-node=4
+#SBATCH --ntasks-per-node=1
+#SBATCH --time=24:00:00
+#SBATCH --output=out/exp_%j_%x.out
+
+# Track4D-360 14B FSDP 288x512 next-stage finetune.
+#
+# Main changes from the 288p backendmix OOM-debug launcher:
+#   - new DATA_TAG / output directory;
+#   - warm-start from the latest existing 288p backendmix checkpoint;
+#   - remove most static datasets from DATA_CONFIG;
+#   - add ReCamMaster CoWTracker + square512 source49 + compact pair1x49
+#     VGGT-Omega labels, q1-only by dataset construction.
+#
+# Submit from repo root on Isambard:
+#   sbatch bash_scripts/track4d_360/ismb/14b/sbatch/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh
+
+export NNODES=8
+export MODEL_SIZE=14b
+export CONDA_ENV="${CONDA_ENV:-th29py312}"
+export HEIGHT=288
+export WIDTH=512
+export NUM_FRAMES=49
+export main_process_port="${main_process_port:-29595}"
+
+export WAN_T2V14B_DIR="${WAN_T2V14B_DIR:-${SLURM_SUBMIT_DIR}/weights/Wan2.1-T2V-14B}"
+export VXF_INIT="${VXF_INIT:-${WAN_T2V14B_DIR}/diffusion_pytorch_model*.safetensors}"
+export MODEL_ID_PREFIX="${MODEL_ID_PREFIX:-Wan-AI/Wan2.1-T2V-14B}"
+export TOKENIZER_PATH="${TOKENIZER_PATH:-${WAN_T2V14B_DIR}/google/umt5-xxl}"
+export T5_PATH="${T5_PATH:-${WAN_T2V14B_DIR}/models_t5_umt5-xxl-enc-bf16.pth}"
+export VAE_PATH="${VAE_PATH:-${WAN_T2V14B_DIR}/Wan2.1_VAE.pth}"
+
+# Prefer the current 288p backendmix run as the next-stage warm start. Fall
+# back to the 384p backendmix/raymap-token parents only when no 288p checkpoint exists.
+export BACKENDMIX_288_PARENT_DIR="${BACKENDMIX_288_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout}"
+export BACKENDMIX_384_PARENT_DIR="${BACKENDMIX_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout}"
+export RAYMAP_384_PARENT_DIR="${RAYMAP_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout}"
+# Warm-start from the 288p backendmix run's step-9000 checkpoint — the requested
+# next-stage starting point. (FSDP1/FSDP2 read it identically: warm-start loads
+# the safetensors BEFORE accelerator.prepare, on the unwrapped model, so it is
+# distributed-backend-independent.) Override WARM_START=... to pick another;
+# falls back to latest backendmix / 384p parents if step-9000 is absent.
+if [ -z "${WARM_START:-}" ]; then
+    if [ -f "${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors" ]; then
+        export WARM_START="${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors"
+    else
+        latest="$(find "${BACKENDMIX_288_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        if [ -z "${latest}" ]; then
+            latest="$(find "${BACKENDMIX_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        if [ -z "${latest}" ]; then
+            latest="$(find "${RAYMAP_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        export WARM_START="${latest:-${RAYMAP_384_PARENT_DIR}/step-3500.safetensors}"
+    fi
+fi
+
+if ! compgen -G "${VXF_INIT}" >/dev/null; then
+    echo "ERROR: VXF_INIT not found: ${VXF_INIT}" >&2
+    exit 1
+fi
+[ -d "${TOKENIZER_PATH}" ] || { echo "ERROR: TOKENIZER_PATH not found: ${TOKENIZER_PATH}" >&2; exit 1; }
+[ -f "${T5_PATH}" ] || { echo "ERROR: T5_PATH not found: ${T5_PATH}" >&2; exit 1; }
+[ -f "${VAE_PATH}" ] || { echo "ERROR: VAE_PATH not found: ${VAE_PATH}" >&2; exit 1; }
+
+export TARGET_EFFECTIVE_BATCH=32
+export gradient_accumulation_steps=1
+export learning_rate="${learning_rate:-2e-5}"
+export save_steps="${save_steps:-500}"
+export dataset_num_workers="${dataset_num_workers:-7}"
+export TRACK4D360_STOP_AFTER_NEXT_SAVE="${TRACK4D360_STOP_AFTER_NEXT_SAVE:-0}"
+
+export dit_finetune_modules="${dit_finetune_modules:-self_attn}"
+export DIT_FINETUNE_TAG="${DIT_FINETUNE_TAG:-attn}"
+export use_plucker_raymap_tokens=1
+export train_plucker_camera_control=0
+export PLUCKER_WANDB_TAG="${PLUCKER_WANDB_TAG:-plucker_raymap_tokens}"
+
+export DATA_CONFIG="${DATA_CONFIG:-${SLURM_SUBMIT_DIR}/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml}"
+export DATA_TAG="${DATA_TAG:-mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512}"
+
+export RECAMMASTER_SOURCE49_ROOT="${RECAMMASTER_SOURCE49_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}"
+export RECAMMASTER_PAIR1X49_LABEL_ROOT="${RECAMMASTER_PAIR1X49_LABEL_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}"
+export RECAMMASTER_COW_ROOT="${RECAMMASTER_COW_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}"
+export TRACK4D360_RECAM_TRACK_SAMPLE_MODE="${TRACK4D360_RECAM_TRACK_SAMPLE_MODE:-grid}"
+export TRACK4D360_RECAM_TRACK_STRIDE="${TRACK4D360_RECAM_TRACK_STRIDE:-2}"
+export TRACK4D360_RECAM_TRACK_SAMPLES="${TRACK4D360_RECAM_TRACK_SAMPLES:-0}"
+# Pair1x49 labels currently cover only the priority block: first 750 numeric
+# train scenes inside each K/focal group (e.g. train/f18_aperture10/scene000..).
+export TRACK4D360_RECAM_SCENE_OFFSET_PER_K="${TRACK4D360_RECAM_SCENE_OFFSET_PER_K:-0}"
+export TRACK4D360_RECAM_SCENE_LIMIT_PER_K="${TRACK4D360_RECAM_SCENE_LIMIT_PER_K:-750}"
+
+# Fixed post-compaction track identity count — MUST be set (the 24h/10k-step
+# backendmix run had this at 65536; this launcher previously omitted it). The
+# ReCamMaster CoW tracks are dense (grid stride 2, samples=0 = uncapped), so
+# without a cap N varies per sample and a heavy draw spikes the track adapter's
+# proj_down activation (B,N,T,hidden) -> the observed CUDA OOM at
+# conv3d_track_adapter.py proj_down (1.16 GiB alloc, step ~770). Fixing N also
+# pads small draws to a constant shape, eliminating per-step fragmentation. Lower
+# to 32768 for extra headroom on contended nodes.
+export TRACK4D360_TRACK_IDENTITY_FIXED_N="${TRACK4D360_TRACK_IDENTITY_FIXED_N:-65536}"
+
+export TRACK4D360_SPARSE_TRACK_PROB="${TRACK4D360_SPARSE_TRACK_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_STRIDE="${TRACK4D360_SPARSE_TRACK_STRIDE:-2|4|8}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB="${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_SIZE="${TRACK4D360_SPARSE_TRACK_BLOCK_SIZE:-64}"
+export TRACK4D360_MULTI_QUERY_PROB="${TRACK4D360_MULTI_QUERY_PROB:-1.0}"
+export TRACK4D360_MULTI_QUERY_CHOICES="${TRACK4D360_MULTI_QUERY_CHOICES:-1|2|3|4}"
+export TRACK4D360_GEOMETRY_NOISE_PROB="${TRACK4D360_GEOMETRY_NOISE_PROB:-1.0}"
+export TRACK4D360_TRACK_OFFSET_STD="${TRACK4D360_TRACK_OFFSET_STD:-0.015}"
+export TRACK4D360_TRACK_TEMPORAL_JITTER_STD="${TRACK4D360_TRACK_TEMPORAL_JITTER_STD:-0.006}"
+export TRACK4D360_TRACK_XY_NOISE_RATIO="${TRACK4D360_TRACK_XY_NOISE_RATIO:-0.25}"
+export TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO="${TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO:-0.5}"
+export TRACK4D360_DEPTH_OFFSET_STD="${TRACK4D360_DEPTH_OFFSET_STD:-0.006}"
+export TRACK4D360_DEPTH_SCALE_JITTER="${TRACK4D360_DEPTH_SCALE_JITTER:-0.01}"
+export TRACK4D360_SOURCE_RGB_NOISE_STD="${TRACK4D360_SOURCE_RGB_NOISE_STD:-0.01}"
+export TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD="${TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD:-0.02}"
+export TRACK4D360_DYNAMIC_MASK_AUG_PROB="${TRACK4D360_DYNAMIC_MASK_AUG_PROB:-0.5}"
+export TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL="${TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL:-5}"
+export TRACK4D360_DYNAMIC_MASK_FLIP_PROB="${TRACK4D360_DYNAMIC_MASK_FLIP_PROB:-0.002}"
+
+export dense_warp_mode="${dense_warp_mode:-hybrid_softsplat}"
+export dense_warp_motion_threshold="${dense_warp_motion_threshold:-0.005}"
+export dense_conditioning_noise_max_level="${dense_conditioning_noise_max_level:-0.15}"
+export source_mask_boundary_depth_dropout_prob="${source_mask_boundary_depth_dropout_prob:-0.5}"
+export source_mask_boundary_kernel_size="${source_mask_boundary_kernel_size:-5}"
+export freeze_track4d_condition_modules="${freeze_track4d_condition_modules:-0}"
+
+export use_gradient_checkpointing=1
+export use_gradient_checkpointing_offload="${use_gradient_checkpointing_offload:-0}"
+export VXF_OFFLOAD_T5_TO_CPU="${VXF_OFFLOAD_T5_TO_CPU:-1}"
+export TRACK4D360_MEM_PROBE="${TRACK4D360_MEM_PROBE:-0}"
+export TRACK4D360_ADAPTER_MEM_PROFILE="${TRACK4D360_ADAPTER_MEM_PROFILE:-0}"
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-${PYTORCH_ALLOC_CONF:-expandable_segments:True}}"
+export PYTORCH_ALLOC_CONF="${PYTORCH_ALLOC_CONF:-${PYTORCH_CUDA_ALLOC_CONF}}"
+export TRACK4D360_DENSE_HYBRID_CACHE_FRAMES="${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES:-7}"
+export TRACK4D360_DENSE_HYBRID_TGT_CHUNK="${TRACK4D360_DENSE_HYBRID_TGT_CHUNK:-2}"
+
+export NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-0}"
+export NCCL_P2P_LEVEL="${NCCL_P2P_LEVEL:-NVL}"
+export NCCL_NVLS_ENABLE=0
+export NCCL_NSOCKS_PERTHREAD=8
+export NCCL_SOCKET_NTHREADS=8
+export NCCL_MAX_NCHANNELS="${NCCL_MAX_NCHANNELS:-32}"
+export NCCL_BUFFSIZE="${NCCL_BUFFSIZE:-8388608}"
+export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
+
+export TRACK4D360_CAPTION_OUTPUT_ROOT="${TRACK4D360_CAPTION_OUTPUT_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output}"
+export SYN4D_CAPTION_DIR="${SYN4D_CAPTION_DIR:-${TRACK4D360_CAPTION_OUTPUT_ROOT}/Syn4D}"
+
+echo "[raymap-tokens 14B pair1x49 next] T2V init: ${VXF_INIT}"
+echo "[raymap-tokens 14B pair1x49 next] warm start: ${WARM_START}"
+echo "[raymap-tokens 14B pair1x49 next] data: ${DATA_CONFIG}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam source49: ${RECAMMASTER_SOURCE49_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam pair1x49 labels: ${RECAMMASTER_PAIR1X49_LABEL_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam CoW: ${RECAMMASTER_COW_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam q1 track sampling: mode=${TRACK4D360_RECAM_TRACK_SAMPLE_MODE}, stride=${TRACK4D360_RECAM_TRACK_STRIDE}, samples=${TRACK4D360_RECAM_TRACK_SAMPLES}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam scene window per K: offset=${TRACK4D360_RECAM_SCENE_OFFSET_PER_K}, limit=${TRACK4D360_RECAM_SCENE_LIMIT_PER_K}"
+echo "[raymap-tokens 14B pair1x49 next] q policy: ReCam q1-only; synthetic choices=${TRACK4D360_MULTI_QUERY_CHOICES}"
+echo "[raymap-tokens 14B pair1x49 next] sparse synthetic tracks: stride_choices=${TRACK4D360_SPARSE_TRACK_STRIDE}, block_jitter_prob=${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB}"
+echo "[raymap-tokens 14B pair1x49 next] trainables: self_attn + raymap Ecam/camera_projector + active track/dense condition modules"
+echo "[raymap-tokens 14B pair1x49 next] freeze track/dense condition modules: ${freeze_track4d_condition_modules}"
+echo "[raymap-tokens 14B pair1x49 next] batch: effective=32, grad_accum=1, lr=${learning_rate}, save_steps=${save_steps}"
+echo "[raymap-tokens 14B pair1x49 next] memory: grad_ckpt_offload=${use_gradient_checkpointing_offload}, hybrid_cache_frames=${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES}, hybrid_tgt_chunk=${TRACK4D360_DENSE_HYBRID_TGT_CHUNK}, cuda_alloc_conf=${PYTORCH_CUDA_ALLOC_CONF}"
+
+source "${SLURM_SUBMIT_DIR}/bash_scripts/track4d_360/ismb/sbatch_common.sh"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml

@@ -0,0 +1,77 @@
+# Next-stage 288x512 backendmix finetune with cleaned ReCamMaster pair1x49 labels.
+#
+# Mix: 35% ReCamMaster pair1x49, 35% Syn4D, 5% Kubric49, 5% Kubric81,
+# and 20% legacy static datasets. The static 20% keeps the previous backendmix
+# relative weights: DL3DV:RE10K:MVS-Synth:TartanAir = 65000:7500:12500:25000.
+# ReCamMaster uses compact square512 source49 + pair1x49 VGGT-Omega scale
+# labels plus CoWTracker source tracks and is q1-only by construction.
+#
+# sbatch_common.sh still sanity-checks for the old MVS-Synth guard string:
+# exclude_sequence_ids=['0000']. This config has no MVS-Synth entry.
+
+num_views: 49
+resolution: [512, 288]
+
+syn4d_root: ${oc.env:SYN4D_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/Syn4D}
+syn4d_caption_dir: ${oc.env:SYN4D_CAPTION_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/Syn4D}
+syn4d_valid_sequences: configs/syn4d_valid_sequences.txt
+dl3dv_root: ${oc.env:DL3DV_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/dl3dv_tar}
+re10k_root: ${oc.env:RE10K_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_re10k}
+mvs_synth_root: ${oc.env:MVS_SYNTH_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_mvs_synth}
+tartanair_root: ${oc.env:TARTANAIR_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_tartanair}
+kubric49_root: ${oc.env:KUBRIC49_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_49frames}
+kubric81_root: ${oc.env:KUBRIC81_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_81frames}
+kubric49_caption_cache_dir: ${oc.env:KUBRIC49_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_49}
+kubric81_caption_cache_dir: ${oc.env:KUBRIC81_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_81}
+recammaster_caption_cache_dir: ${oc.env:RECAMMASTER_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/recammaster}
+recammaster_root: ${oc.env:RECAMMASTER_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset}
+multicam_fg_mask_root: ${oc.env:MULTICAM_FG_MASK_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_SAM3_FG_81f_sources_person_human_288x512}
+recammaster_source49_root: ${oc.env:RECAMMASTER_SOURCE49_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}
+recammaster_pair1x49_label_root: ${oc.env:RECAMMASTER_PAIR1X49_LABEL_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}
+recammaster_cow_root: ${oc.env:RECAMMASTER_COW_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}
+recammaster_track_sample_mode: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLE_MODE,grid}
+recammaster_track_stride: ${oc.env:TRACK4D360_RECAM_TRACK_STRIDE,2}
+recammaster_track_samples: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLES,0}
+recammaster_scene_offset_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_OFFSET_PER_K,0}
+recammaster_scene_limit_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_LIMIT_PER_K,750}
+
+sparse_track_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_PROB,1.0}
+sparse_track_stride: ${oc.env:TRACK4D360_SPARSE_TRACK_STRIDE,2|4|8}
+sparse_track_query_stride_mode: ${oc.env:TRACK4D360_SPARSE_TRACK_QUERY_STRIDE_MODE,query_count_floor}
+sparse_track_block_jitter_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB,1.0}
+sparse_track_block_size: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_SIZE,64}
+multi_query_prob: ${oc.env:TRACK4D360_MULTI_QUERY_PROB,1.0}
+multi_query_choices: ${oc.env:TRACK4D360_MULTI_QUERY_CHOICES,1|2|3|4}
+geometry_noise_prob: ${oc.env:TRACK4D360_GEOMETRY_NOISE_PROB,1.0}
+track_offset_std: ${oc.env:TRACK4D360_TRACK_OFFSET_STD,0.015}
+track_temporal_jitter_std: ${oc.env:TRACK4D360_TRACK_TEMPORAL_JITTER_STD,0.006}
+track_xy_noise_ratio: ${oc.env:TRACK4D360_TRACK_XY_NOISE_RATIO,0.25}
+track_frame_depth_jitter_ratio: ${oc.env:TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO,0.5}
+depth_offset_std: ${oc.env:TRACK4D360_DEPTH_OFFSET_STD,0.006}
+depth_scale_jitter: ${oc.env:TRACK4D360_DEPTH_SCALE_JITTER,0.01}
+source_rgb_noise_std: ${oc.env:TRACK4D360_SOURCE_RGB_NOISE_STD,0.01}
+source_rgb_frame_jitter_std: ${oc.env:TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD,0.02}
+dynamic_mask_aug_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_AUG_PROB,0.5}
+dynamic_mask_morph_kernel: ${oc.env:TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL,5}
+dynamic_mask_flip_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_FLIP_PROB,0.002}
+
+train_datasets:
+  # 35% ReCamMaster CoWTracker + square512 source49 + compact pair1x49 VGGT labels.
+  # This dataset is q1 only: it always emits track_query_indices=[0]. The
+  # default track density is grid stride 2 to avoid lifting every CoW pixel
+  # during training; override TRACK4D360_RECAM_TRACK_SAMPLE_MODE=all for dense.
+  - "87500 @ ReCamMasterCowVGGT_NVS(ROOT='${recammaster_root}', vggt_root='${recammaster_source49_root}', source_vggt_root='${recammaster_source49_root}', pair1x49_label_root='${recammaster_pair1x49_label_root}', geometry_source='pair1x49_label_scale_only', cow_root='${recammaster_cow_root}', fg_mask_root='${multicam_fg_mask_root}', caption_cache_dir='${recammaster_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, vggt_post_crop_mode='cow_aspect', rgb_post_crop_mode='cow_aspect', track_sample_mode='${recammaster_track_sample_mode}', track_stride=${recammaster_track_stride}, track_samples=${recammaster_track_samples}, scene_offset_per_k=${recammaster_scene_offset_per_k}, scene_limit_per_k=${recammaster_scene_limit_per_k})"
+  # 35% Syn4D dynamic multi-camera mesh-tracked 3D.
+  - "87500 @ GeometryRobustnessWrapper_NVS(Syn4D_Multi_NVS(ROOT='${syn4d_root}', caption_dir='${syn4d_caption_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, track_spatial_stride=1, exclude_scenes=['flying_group'], valid_sequences_file='${syn4d_valid_sequences}', use_mp4_rgb=True, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 49-frame dynamic data.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric49_root}', caption_cache_dir='${kubric49_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 81-frame dynamic data sampled to 49.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric81_root}', caption_cache_dir='${kubric81_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 11.82% DL3DV (real, DA3 depth). Static total is 20%; this keeps the old static relative weight.
+  - "29550 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${dl3dv_root}', dataset_type='dl3dv', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=29550, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=0.5, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, use_da3_depth=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 1.38% RE10K (real, DA3 depth).
+  - "3450 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${re10k_root}', dataset_type='re10k', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=3450, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 2.28% MVS-Synth (synthetic, GT depth; scene 0000 held out).
+  - "5700 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${mvs_synth_root}', dataset_type='mvs_synth', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=5700, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, exclude_sequence_ids=['0000']), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 4.52% TartanAir (static forward-moving scenes).
+  - "11300 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${tartanair_root}', dataset_type='tartanair', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=11300, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=1.0, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/torch_profiler/rank0.1780383301809004862.pt.trace.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddbf7c30a739217186bd8eac8508c6c9bf190659e87eb62aa65e8258afa93f8c
+size 4715028167

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/debug-internal.log

@@ -0,0 +1,95 @@
+{"time":"2026-06-02T06:50:20.584631685Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T06:50:20.58696336Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T06:50:20.969725714Z","level":"INFO","msg":"stream: created new stream","id":"3241szoo"}
+{"time":"2026-06-02T06:50:20.96994734Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T06:50:20.970735157Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T06:50:20.9707597Z","level":"INFO","msg":"writer: started","stream_id":"3241szoo"}
+{"time":"2026-06-02T06:50:20.971093546Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T06:50:22.923197858Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T06:50:23.133364725Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:30.303531233Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T06:50:30.305530822Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":47}
+{"time":"2026-06-02T06:50:37.923464724Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":115,"uploaded_len":2}
+{"time":"2026-06-02T06:50:38.126829678Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:52.923591564Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T06:50:53.109219757Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:07.924029832Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-06-02T06:51:08.114725101Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:22.92352313Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-06-02T06:51:23.119009653Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:37.924474754Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-06-02T06:51:38.141711785Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:52.923701659Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":116,"console_lines":1}
+{"time":"2026-06-02T06:51:53.111102047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:07.924313952Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":117,"console_lines":3}
+{"time":"2026-06-02T06:52:08.119941045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:22.924342752Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:52:23.14398364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:37.924171237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":2,"events_offset":16,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:52:38.127651722Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:52.923475326Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:52:53.111875741Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:07.924090817Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:08.123142555Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:22.924082144Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":22,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:23.155915012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:37.924127613Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:38.132051519Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:52.924552016Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:53.117190814Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:07.923825475Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:54:08.143248291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:22.924253036Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T06:54:23.162918899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:37.924269819Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T06:54:38.178293682Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:52.92343901Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-06-02T06:54:53.126764907Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:07.924320537Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-06-02T06:55:08.132013064Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:22.924083496Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-06-02T06:55:23.183084706Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:37.923701069Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-06-02T06:55:38.176471756Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:52.924750892Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:55:53.115223076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:07.923910025Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:08.170806645Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:22.924071465Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:23.147054865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:37.923990093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:38.174203624Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:52.924661661Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:53.139214445Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:07.924453061Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:08.169695656Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:22.924163918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":54,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:23.146405546Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:37.924201145Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:38.109290006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:52.925187457Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:53.15045367Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:07.924653715Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":60,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:08.180545128Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:22.923699153Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:23.120052374Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:37.923835946Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:38.182881949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:52.923692065Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:53.140186235Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:07.924480574Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":2,"events_offset":68,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:08.135037406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:22.923975937Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":70,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:23.116919373Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:37.924640345Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:38.187698331Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:52.92428249Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:53.247046902Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:57.467547914Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T06:59:57.467952574Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":120,"console_lines":92,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T06:59:57.66002893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:57.660797388Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T06:59:57.665684932Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T06:59:57.665718499Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T06:59:57.666134903Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T06:59:57.66616015Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 06:50:19,995 INFO    MainThread:248506 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_setup.py:_flush():81] Configure stats pid to 248506
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/logs/debug.log
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/logs/debug-internal.log
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:init():848] calling init triggers
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 0, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:init():896] starting backend
+2026-06-02 06:50:20,574 INFO    MainThread:248506 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 06:50:20,582 INFO    MainThread:248506 [wandb_init.py:init():919] backend started and connected
+2026-06-02 06:50:20,584 INFO    MainThread:248506 [wandb_init.py:init():989] updated telemetry
+2026-06-02 06:50:20,629 INFO    MainThread:248506 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 06:50:21,612 INFO    MainThread:248506 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 06:50:22,909 INFO    MainThread:248506 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 06:50:22,909 INFO    MainThread:248506 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 06:50:22,909 INFO    MainThread:248506 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 06:50:22,910 INFO    MainThread:248506 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 06:50:22,916 INFO    MainThread:248506 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 06:59:56,752 INFO    MainThread:248506 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/3241szoo
+2026-06-02 06:59:56,772 INFO    MainThread:248506 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 06:59:56,773 INFO    MainThread:248506 [wandb_run.py:_restore():2490] restore
+2026-06-02 06:59:56,773 INFO    MainThread:248506 [wandb_run.py:_restore():2496] restore done
+2026-06-02 06:59:57,664 INFO    MainThread:248506 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/files/config.yaml

@@ -0,0 +1,700 @@
+_wandb:
+    value:
+        cli_version: 0.26.0
+        e:
+            4n8ybj8xil1hl68ckg9qyr4h3ok6yfl3:
+                args:
+                    - --model_id_with_origin_paths
+                    - Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+                    - --tokenizer_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+                    - --task
+                    - sft:v2v
+                    - --model_variant
+                    - vt2v
+                    - --v2v_concat_mode
+                    - width
+                    - --height
+                    - "288"
+                    - --width
+                    - "512"
+                    - --num_frames
+                    - "49"
+                    - --track4d360_data_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+                    - --steps_per_epoch
+                    - "25000"
+                    - --num_epochs
+                    - "200"
+                    - --learning_rate
+                    - "2e-5"
+                    - --gradient_accumulation_steps
+                    - "1"
+                    - --save_steps
+                    - "500"
+                    - --remove_prefix_in_ckpt
+                    - pipe.dit.
+                    - --trainable_models
+                    - dit.domain_embedding,dit.control_patch_embedding
+                    - --dit_finetune_modules
+                    - self_attn
+                    - --track_injection_mode
+                    - per_block
+                    - --track_injection_block_mode
+                    - concat_project
+                    - --track_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml
+                    - --train_sigma_shift
+                    - "5.0"
+                    - --use_query_frame_impulse_condition
+                    - --use_plucker_camera_control
+                    - --enable_v2v_plucker_camera_control
+                    - --use_plucker_raymap_tokens
+                    - --use_perlayer_camera_control
+                    - --camera_injection_mode
+                    - projector
+                    - --camera_plucker_encoder_mode
+                    - patchify
+                    - --use_dense_branch
+                    - --dense_in_channels
+                    - "5"
+                    - --dense_proj_dim
+                    - "32"
+                    - --dense_num_residual_blocks
+                    - "0"
+                    - --dense_alpha_track
+                    - "1.0"
+                    - --dense_conditioning_noise_max_level
+                    - "0.15"
+                    - --source_mask_boundary_depth_dropout_prob
+                    - "0.5"
+                    - --source_mask_boundary_kernel_size
+                    - "5"
+                    - --dense_warp_mode
+                    - hybrid_softsplat
+                    - --dense_warp_motion_threshold
+                    - "0.005"
+                    - --cond_drop_geo_prob
+                    - "0.1"
+                    - --cond_drop_track_prob
+                    - "0.1"
+                    - --cond_drop_dense_prob
+                    - "0.0"
+                    - --initial_step
+                    - "2530"
+                    - --output_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout
+                    - --dataset_num_workers
+                    - "7"
+                    - --dataloader_prefetch_factor
+                    - "2"
+                    - --drop_text_prob
+                    - "0.1"
+                    - --debug_latents_every
+                    - "500"
+                    - --allow_existing_checkpoints
+                    - --wandb_project
+                    - track4d-360
+                    - --wandb_name
+                    - ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp
+                    - --wandb_tags
+                    - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+                    - --use_wandb
+                    - --vxf_init_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+                    - --use_gradient_checkpointing
+                    - --resume_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+                    - --max_train_steps
+                    - "2560"
+                    - --torch_profile_active_steps
+                    - "5"
+                    - --torch_profile_wait_steps
+                    - "3"
+                    - --torch_profile_warmup_steps
+                    - "1"
+                    - --torch_profile_repeat
+                    - "1"
+                    - --no_torch_profile_record_shapes
+                    - --no_torch_profile_memory
+                codePath: scripts/wan2.1_fun/train_track4d360.py
+                codePathLocal: scripts/wan2.1_fun/train_track4d360.py
+                cpu_count: 288
+                cpu_count_logical: 288
+                cudaVersion: "12.7"
+                disk:
+                    /:
+                        total: "255795724288"
+                        used: "527630336"
+                email: yushi.lan@outlook.com
+                executable: /projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12
+                git:
+                    commit: b6681ed2127310d14bc5b93ae5f1d20df8b9e907
+                    remote: git@github.com:NIRVANALAN/track4d-360.git
+                gpu: NVIDIA GH200 120GB
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-5728e766-d093-23d3-0fed-a94feb031fe3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-098a504b-2d3b-7b84-0901-6fbce487448c
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-4752a23e-8a0b-30b8-6dca-8c3bedf9c98c
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-c54beba8-b9e0-3543-a467-81ae596f05f6
+                host: nid010552
+                memory:
+                    total: "919613407232"
+                os: Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38
+                program: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py
+                python: CPython 3.12.13
+                root: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout
+                slurm:
+                    cluster_name: gracehopper
+                    conf: /var/spool/slurmd/conf-cache/slurm.conf
+                    cpu_bind: quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_list: 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_type: 'mask_cpu:'
+                    cpu_bind_verbose: quiet
+                    cpus_on_node: "288"
+                    distribution: cyclic
+                    gpus: "32"
+                    gpus_on_node: "4"
+                    gpus_per_node: "4"
+                    gtids: "0"
+                    job_account: brics.u6ew
+                    job_cpus_per_node: 288(x8)
+                    job_end_time: "1780389885"
+                    job_gid: "1483804847"
+                    job_gpus: 0,1,2,3
+                    job_id: "4982943"
+                    job_name: 14b_pair1x49_profile_ablate_bothres0_fsdp2
+                    job_nodelist: nid[010552-010553,010558,010561-010565]
+                    job_num_nodes: "8"
+                    job_partition: workq
+                    job_qos: normal
+                    job_start_time: "1780382685"
+                    job_uid: "1483804847"
+                    job_user: yushi.u6ew
+                    jobid: "4982943"
+                    launch_node_ipaddr: 10.242.15.10
+                    localid: "0"
+                    mpi_type: pmi2
+                    nnodes: "8"
+                    nodeid: "0"
+                    nodelist: nid[010552-010553,010558,010561-010565]
+                    nprocs: "8"
+                    ntasks: "8"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    srun_comm_host: 10.242.15.10
+                    srun_comm_port: "41219"
+                    step_gpus: 0,1,2,3
+                    step_id: "0"
+                    step_launcher_port: "41219"
+                    step_nodelist: nid[010552-010553,010558,010561-010565]
+                    step_num_nodes: "8"
+                    step_num_tasks: "8"
+                    step_resv_ports: 26620-26621
+                    step_tasks_per_node: 1(x8)
+                    stepid: "0"
+                    submit_dir: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360
+                    submit_host: login40
+                    task_pid: "248425"
+                    tasks_per_node: 1(x8)
+                    topology_addr: global.group7.nid010552
+                    topology_addr_pattern: switch.switch.node
+                    umask: "0022"
+                startedAt: "2026-06-02T06:50:19.986166Z"
+                writerId: 4n8ybj8xil1hl68ckg9qyr4h3ok6yfl3
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 98
+                - 106
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+                - 106
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 41
+                - 61
+            "4": 3.12.13
+            "5": 0.26.0
+            "6": 4.46.2
+            "12": 0.26.0
+            "13": linux-aarch64
+allow_existing_checkpoints:
+    value: true
+camera_condition_cfg_dropout_prob:
+    value: 0
+camera_injection_mode:
+    value: projector
+camera_plucker_encoder_mode:
+    value: patchify
+channel_concat_stage:
+    value: null
+cond_drop_dense_prob:
+    value: 0
+cond_drop_geo_prob:
+    value: 0.1
+cond_drop_track_prob:
+    value: 0.1
+data_file_keys:
+    value: image,video
+dataloader_prefetch_factor:
+    value: 2
+dataset_base_path:
+    value: null
+dataset_metadata_path:
+    value: null
+dataset_num_workers:
+    value: 7
+dataset_repeat:
+    value: 1
+debug_latents:
+    value: false
+debug_latents_every:
+    value: 500
+debug_latents_fps:
+    value: 16
+debug_latents_max_frames:
+    value: null
+debug_latents_outdir:
+    value: debug_latents
+debug_latents_quality:
+    value: 5
+debug_latents_vae_tile_size:
+    value:
+        - 30
+        - 52
+debug_latents_vae_tile_stride:
+    value:
+        - 15
+        - 26
+debug_latents_vae_tiled:
+    value: true
+debug_missing_grads:
+    value: false
+dense_alpha_track:
+    value: 1
+dense_conditioning_noise_max_level:
+    value: 0.15
+dense_hybrid_disable_dynamic_masks:
+    value: false
+dense_in_channels:
+    value: 5
+dense_num_residual_blocks:
+    value: 0
+dense_only_condition_modules:
+    value: false
+dense_proj_dim:
+    value: 32
+dense_warp_mode:
+    value: hybrid_softsplat
+dense_warp_motion_threshold:
+    value: 0.005
+disable_track_modules:
+    value: false
+dit_finetune_modules:
+    value: self_attn
+drop_i2v_prob:
+    value: null
+drop_t2v_prob:
+    value: null
+drop_text_prob:
+    value: 0.1
+drop_video_prob:
+    value: 0
+dummy:
+    value: false
+enable_v2v_plucker_camera_control:
+    value: true
+explicit_motion_loss_decode_checkpoint:
+    value: false
+explicit_motion_loss_decode_chunk_size:
+    value: 0
+explicit_motion_loss_snr_sigmoid:
+    value: true
+explicit_motion_loss_snr_sigmoid_a:
+    value: 1
+explicit_motion_loss_snr_sigmoid_b:
+    value: -5
+explicit_motion_loss_snr_weight:
+    value: true
+explicit_motion_loss_weight:
+    value: 0
+extra_inputs:
+    value: null
+find_unused_parameters:
+    value: false
+fixed_num_inference_steps:
+    value: 50
+fixed_sigma:
+    value: null
+fixed_step_frac:
+    value: null
+fp8_models:
+    value: null
+freeze_lora_after_load:
+    value: false
+freeze_track_block_injector:
+    value: false
+freeze_track4d_condition_modules:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gs_abs_depth_loss_weight:
+    value: 0
+gs_assoc_track_2d_loss_weight:
+    value: 0
+gs_depth_loss_weight:
+    value: 0.05
+gs_direct_pos_loss_weight:
+    value: 0
+gs_disable_delta_opacity:
+    value: false
+gs_dyn_depth_loss_weight:
+    value: 0.5
+gs_dynamic_max_t_video:
+    value: 64
+gs_dynamic_mode:
+    value: static
+gs_feat_dim:
+    value: 1024
+gs_head_init_checkpoint:
+    value: null
+gs_head_style:
+    value: flashworld
+gs_legacy_union_render:
+    value: false
+gs_loss_weight:
+    value: 1
+gs_lpips_loss_weight:
+    value: 0.5
+gs_lyra_depth_param:
+    value: exp
+gs_lyra_dfar:
+    value: 10
+gs_lyra_dnear:
+    value: 0.1
+gs_lyra_log_depth_shift:
+    value: 0
+gs_mse_loss_weight:
+    value: 1
+gs_num_points_per_pixel:
+    value: 2
+gs_only_training:
+    value: false
+gs_opa_tv_weight:
+    value: 0.01
+gs_opacity_loss_weight:
+    value: 0.1
+gs_opacity_prune_keep_frac:
+    value: 1
+gs_pos_loss_weight:
+    value: 1
+gs_predict_rotation:
+    value: false
+gs_render_patch_size:
+    value: null
+gs_replay_cache_size:
+    value: 0
+gs_rgb_loss_weight:
+    value: null
+gs_scale_cap:
+    value: null
+gs_sh_degree:
+    value: 2
+gs_track_accel_loss_weight:
+    value: 0
+gs_track_canonical_detach:
+    value: false
+gs_track_canonical_loss_weight:
+    value: 0
+gs_track_delta_loss_weight:
+    value: 0
+gs_track_final_pos_loss_weight:
+    value: 0
+gs_track_joint_render:
+    value: false
+gs_track_loss_quantile:
+    value: 0.98
+gs_track_supervision_mode:
+    value: pointmap
+gs_xyz_tv_weight:
+    value: 0.01
+gs_zero_features:
+    value: false
+height:
+    value: 288
+initial_step:
+    value: 2530
+initialize_model_on_cpu:
+    value: false
+input_video_noise_random_max:
+    value: 500
+input_video_noise_strength:
+    value: 0
+joint_depth_prediction:
+    value: false
+joint_track_depth_prediction:
+    value: false
+joint_track_prediction:
+    value: false
+learning_rate:
+    value: 2e-05
+lora_alpha:
+    value: null
+lora_base_model:
+    value: null
+lora_checkpoint:
+    value: null
+lora_rank:
+    value: 32
+lora_target_modules:
+    value: q,k,v,o,ffn.0,ffn.2
+max_checkpoints_to_keep:
+    value: 5
+max_pixels:
+    value: 1048576
+max_train_steps:
+    value: 2560
+model_id_with_origin_paths:
+    value: Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+model_paths:
+    value: null
+model_variant:
+    value: vt2v
+multi_frame_motion_threshold:
+    value: 0.01
+multi_frame_warped_appearance:
+    value: false
+no_gs_latent_detach:
+    value: false
+normalize_track_with_xyz_cond_stats:
+    value: false
+num_epochs:
+    value: 200
+num_frames:
+    value: 49
+num_workers:
+    value: 7
+offload_models:
+    value: null
+output_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout
+plucker_use_ray_origin:
+    value: false
+pointmap_stats_path:
+    value: null
+predict_track_residual:
+    value: false
+prefetch_factor:
+    value: 2
+prenormalized_track_video_signed_log:
+    value: false
+preset_lora_model:
+    value: null
+preset_lora_path:
+    value: null
+remove_prefix_in_ckpt:
+    value: pipe.dit.
+residual_linear_normalization:
+    value: false
+residual_log_scale_factor:
+    value: 3
+residual_target_xyz_source:
+    value: clean
+residual_zero_center_normalization:
+    value: false
+resume_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+save_every_n_epochs:
+    value: 2
+save_steps:
+    value: 500
+scene:
+    value: null
+sequence_idx:
+    value: null
+skip_diffusion_loss:
+    value: false
+skip_v2v_required_trainables:
+    value: false
+source_mask_boundary_depth_dropout_prob:
+    value: 0.5
+source_mask_boundary_kernel_size:
+    value: 5
+steps_per_epoch:
+    value: 25000
+stream3r_common_scale_stat:
+    value: mean
+syn4d_root:
+    value: /scratch/shared/beegfs/zeren/Syn4D
+target_depth_log_cap:
+    value: 14
+target_depth_max_disparity:
+    value: 5
+target_depth_norm_type:
+    value: log_depth
+task:
+    value: sft:v2v
+tokenizer_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+torch_distributed_debug:
+    value: null
+torch_profile_active_steps:
+    value: 5
+torch_profile_dir:
+    value: null
+torch_profile_memory:
+    value: false
+torch_profile_record_shapes:
+    value: false
+torch_profile_repeat:
+    value: 1
+torch_profile_wait_steps:
+    value: 3
+torch_profile_warmup_steps:
+    value: 1
+torch_profile_with_stack:
+    value: false
+track_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml
+track_injection_block_mode:
+    value: concat_project
+track_injection_mode:
+    value: per_block
+track_spatial_stride:
+    value: 1
+track_xyz_normalization:
+    value: minmax
+track_xyz_quantile_high:
+    value: 0.98
+track_xyz_quantile_low:
+    value: 0.02
+track4d360_data_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+train_plucker_camera_control:
+    value: false
+train_sigma_shift:
+    value: 5
+trainable_models:
+    value: dit.domain_embedding,dit.control_patch_embedding
+use_dense_branch:
+    value: true
+use_dense_null_token_fallback:
+    value: false
+use_depth_null_token:
+    value: false
+use_gen3c_warp_latent_channels:
+    value: false
+use_gradient_checkpointing:
+    value: true
+use_gradient_checkpointing_offload:
+    value: false
+use_gs_feedback_training:
+    value: false
+use_gs_output:
+    value: false
+use_perlayer_camera_control:
+    value: true
+use_plucker_camera_control:
+    value: true
+use_plucker_raymap_tokens:
+    value: true
+use_query_frame_impulse_condition:
+    value: true
+use_residual_tracks:
+    value: true
+use_track_null_token_fallback:
+    value: false
+use_vista4d_latent_streams:
+    value: false
+use_wandb:
+    value: true
+use_xyz_video_condition:
+    value: false
+v2v_channel_patch_embed_mode:
+    value: add
+v2v_concat_mode:
+    value: width
+vxf_init_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+wandb_entity:
+    value: null
+wandb_log_freq:
+    value: 100
+wandb_log_gradients:
+    value: false
+wandb_log_parameters:
+    value: false
+wandb_name:
+    value: ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp
+wandb_notes:
+    value: null
+wandb_project:
+    value: track4d-360
+wandb_tags:
+    value:
+        - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+wandb_watch_model:
+    value: false
+warped_appearance_fusion:
+    value: concat_proj
+warped_condition_mode:
+    value: "off"
+warped_geom_only_dense:
+    value: true
+weight_decay:
+    value: 0.01
+width:
+    value: 512
+xyz_video_condition_aug_prob:
+    value: 0
+xyz_video_condition_cfg_dropout_prob:
+    value: 0
+xyz_video_condition_dropout_prob:
+    value: 0
+xyz_video_condition_frame_scale_std:
+    value: 0
+xyz_video_condition_mode:
+    value: xyz
+xyz_video_condition_noise_std:
+    value: 0
+xyz_video_condition_scale:
+    value: 1
+xyz_video_condition_z_noise_std:
+    value: 0

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/files/output.log

@@ -0,0 +1,212 @@
+✓ WandB initialized: https://wandb.ai/nirvanalan/track4d-360/runs/3241szoo
+/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/lib/python3.12/site-packages/accelerate/utils/fsdp_utils.py:742: UserWarning: FSDP upcast of low precision parameters to fp32 (since mixed_precision != 'no') may affect the precision of model checkpoints. This effects 585 parameters: ['pipe.dit.domain_embedding', 'pipe.dit.blocks.0.self_attn.q.weight', 'pipe.dit.blocks.0.self_attn.q.bias', 'pipe.dit.blocks.0.self_attn.k.weight', 'pipe.dit.blocks.0.self_attn.k.bias', 'pipe.dit.blocks.0.self_attn.v.weight', 'pipe.dit.blocks.0.self_attn.v.bias', 'pipe.dit.blocks.0.self_attn.o.weight', 'pipe.dit.blocks.0.self_attn.o.bias', 'pipe.dit.blocks.0.self_attn.norm_q.weight', 'pipe.dit.blocks.0.self_attn.norm_k.weight', 'pipe.dit.blocks.0.camera_projector.weight', 'pipe.dit.blocks.0.camera_projector.bias', 'pipe.dit.blocks.1.self_attn.q.weight', 'pipe.dit.blocks.1.self_attn.q.bias', 'pipe.dit.blocks.1.self_attn.k.weight', 'pipe.dit.blocks.1.self_attn.k.bias', 'pipe.dit.blocks.1.self_attn.v.weight', 'pipe.dit.blocks.1.self_attn.v.bias', 'pipe.dit.blocks.1.self_attn.o.weight', 'pipe.dit.blocks.1.self_attn.o.bias', 'pipe.dit.blocks.1.self_attn.norm_q.weight', 'pipe.dit.blocks.1.self_attn.norm_k.weight', 'pipe.dit.blocks.1.camera_projector.weight', 'pipe.dit.blocks.1.camera_projector.bias', 'pipe.dit.blocks.2.self_attn.q.weight', 'pipe.dit.blocks.2.self_attn.q.bias', 'pipe.dit.blocks.2.self_attn.k.weight', 'pipe.dit.blocks.2.self_attn.k.bias', 'pipe.dit.blocks.2.self_attn.v.weight', 'pipe.dit.blocks.2.self_attn.v.bias', 'pipe.dit.blocks.2.self_attn.o.weight', 'pipe.dit.blocks.2.self_attn.o.bias', 'pipe.dit.blocks.2.self_attn.norm_q.weight', 'pipe.dit.blocks.2.self_attn.norm_k.weight', 'pipe.dit.blocks.2.camera_projector.weight', 'pipe.dit.blocks.2.camera_projector.bias', 'pipe.dit.blocks.3.self_attn.q.weight', 'pipe.dit.blocks.3.self_attn.q.bias', 'pipe.dit.blocks.3.self_attn.k.weight', 'pipe.dit.blocks.3.self_attn.k.bias', 'pipe.dit.blocks.3.self_attn.v.weight', 'pipe.dit.blocks.3.self_attn.v.bias', 'pipe.dit.blocks.3.self_attn.o.weight', 'pipe.dit.blocks.3.self_attn.o.bias', 'pipe.dit.blocks.3.self_attn.norm_q.weight', 'pipe.dit.blocks.3.self_attn.norm_k.weight', 'pipe.dit.blocks.3.camera_projector.weight', 'pipe.dit.blocks.3.camera_projector.bias', 'pipe.dit.blocks.4.self_attn.q.weight', 'pipe.dit.blocks.4.self_attn.q.bias', 'pipe.dit.blocks.4.self_attn.k.weight', 'pipe.dit.blocks.4.self_attn.k.bias', 'pipe.dit.blocks.4.self_attn.v.weight', 'pipe.dit.blocks.4.self_attn.v.bias', 'pipe.dit.blocks.4.self_attn.o.weight', 'pipe.dit.blocks.4.self_attn.o.bias', 'pipe.dit.blocks.4.self_attn.norm_q.weight', 'pipe.dit.blocks.4.self_attn.norm_k.weight', 'pipe.dit.blocks.4.camera_projector.weight', 'pipe.dit.blocks.4.camera_projector.bias', 'pipe.dit.blocks.5.self_attn.q.weight', 'pipe.dit.blocks.5.self_attn.q.bias', 'pipe.dit.blocks.5.self_attn.k.weight', 'pipe.dit.blocks.5.self_attn.k.bias', 'pipe.dit.blocks.5.self_attn.v.weight', 'pipe.dit.blocks.5.self_attn.v.bias', 'pipe.dit.blocks.5.self_attn.o.weight', 'pipe.dit.blocks.5.self_attn.o.bias', 'pipe.dit.blocks.5.self_attn.norm_q.weight', 'pipe.dit.blocks.5.self_attn.norm_k.weight', 'pipe.dit.blocks.5.camera_projector.weight', 'pipe.dit.blocks.5.camera_projector.bias', 'pipe.dit.blocks.6.self_attn.q.weight', 'pipe.dit.blocks.6.self_attn.q.bias', 'pipe.dit.blocks.6.self_attn.k.weight', 'pipe.dit.blocks.6.self_attn.k.bias', 'pipe.dit.blocks.6.self_attn.v.weight', 'pipe.dit.blocks.6.self_attn.v.bias', 'pipe.dit.blocks.6.self_attn.o.weight', 'pipe.dit.blocks.6.self_attn.o.bias', 'pipe.dit.blocks.6.self_attn.norm_q.weight', 'pipe.dit.blocks.6.self_attn.norm_k.weight', 'pipe.dit.blocks.6.camera_projector.weight', 'pipe.dit.blocks.6.camera_projector.bias', 'pipe.dit.blocks.7.self_attn.q.weight', 'pipe.dit.blocks.7.self_attn.q.bias', 'pipe.dit.blocks.7.self_attn.k.weight', 'pipe.dit.blocks.7.self_attn.k.bias', 'pipe.dit.blocks.7.self_attn.v.weight', 'pipe.dit.blocks.7.self_attn.v.bias', 'pipe.dit.blocks.7.self_attn.o.weight', 'pipe.dit.blocks.7.self_attn.o.bias', 'pipe.dit.block
+  warnings.warn(
+
+================================================================================
+TRAINABLE PARAMETERS VERIFICATION
+================================================================================
+
+Trainable modules (93 module groups):
+
+  📦 pipe.dit.blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+     pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.dit._camera_patchify_conv:
+     Total params: 36,870
+         ├─ weight: 36,864
+         ├─ bias: 6
+
+  📦 pipe.dit._camera_plucker_patchify_encoder:
+     Total params: 35,840
+         ├─ weight: 30,720
+         ├─ bias: 5,120
+
+  📦 pipe.dit.control_patch_embedding:
+     Total params: 332,800
+         ├─ weight: 327,680
+         ├─ bias: 5,120
+
+  📦 pipe.dit.domain_embedding:
+     Total params: 10,240
+     pipe.dit.domain_embedding: 10,240
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+     pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         ├─ 16.weight: 27,013,120
+         ├─ 17.weight: 27,013,120
+         ├─ 18.weight: 27,013,120
+         ├─ 19.weight: 27,013,120
+         ├─ 2.weight: 27,013,120
+         ├─ 20.weight: 27,013,120
+         ├─ 21.weight: 27,013,120
+         ├─ 22.weight: 27,013,120
+         ├─ 23.weight: 27,013,120
+         ├─ 24.weight: 27,013,120
+         ├─ 25.weight: 27,013,120
+         ├─ 26.weight: 27,013,120
+         ├─ 27.weight: 27,013,120
+         ├─ 28.weight: 27,013,120
+         ├─ 29.weight: 27,013,120
+         ├─ 3.weight: 27,013,120
+         ├─ 30.weight: 27,013,120
+         ├─ … (55 more)
+
+────────────────────────────────────────────────────────────────────────────────
+NOTE: FSDP is active; counts below are from the rank-local post-wrap parameter view, not a full global parameter census.
+Total trainable parameters: 6,325,630,450 (6325.63M)
+Total frozen parameters:    10,363,571,200 (10363.57M)
+Percentage trainable (overall): 37.90%
+Percentage trainable (in requested modules: dit): 33.60%
+================================================================================
+
+[torch-profiler] rank0 enabled (wait=3, warmup=1, active=5, repeat=1) -> /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/torch_profiler
+Epoch 1/200:   0%|          | 29/7813 [09:09<41:00:07, 18.96s/it, opt_step=2559, loss=0.0660]  
+[Track4D-360] First train batch runtime flags: use_gradient_checkpointing=True, use_gradient_checkpointing_offload=False
+[soft_splat_warp] WARN: tracks grid 160x294 (N=47040) != image grid 288x512 (N=147456); bilinear-resampling tracks_world to image grid. This warning fires once per process.
+[track4d-runtime] rank0 patched DiTBlock.forward saw mode=track block_idx=0
+[Attention] Using: Flash Attention 2 (flash_attn)
+
+[Training] Reached max_train_steps=2560. Stopping training.
+
+[Checkpoint] Saving 585 parameters (total 6,325,630,450 ≈ 6325.63M):
+
+  📦 blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 _camera_patchify_conv.bias:
+     Total params: 6
+         _camera_patchify_conv.bias: 6
+
+  📦 _camera_patchify_conv.weight:
+     Total params: 36,864
+         _camera_patchify_conv.weight: 36,864
+
+  📦 _camera_plucker_patchify_encoder.bias:
+     Total params: 5,120
+         _camera_plucker_patchify_encoder.bias: 5,120
+
+  📦 _camera_plucker_patchify_encoder.weight:
+     Total params: 30,720
+         _camera_plucker_patchify_encoder.weight: 30,720
+
+  📦 control_patch_embedding.bias:
+     Total params: 5,120
+         control_patch_embedding.bias: 5,120
+
+  📦 control_patch_embedding.weight:
+     Total params: 327,680
+         control_patch_embedding.weight: 327,680
+
+  📦 domain_embedding:
+     Total params: 10,240
+         domain_embedding: 10,240
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+         pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+         pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         … (72 more)

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/files/requirements.txt

@@ -0,0 +1,208 @@
+videox-fun==1.0.1
+track4d-360==0.1.0
+func_timeout==4.3.5
+click==8.3.2
+pip==26.0.1
+nvidia-cuda-nvrtc-cu12==12.6.77
+ImageIO==2.37.3
+numpy==1.26.4
+boto3==1.42.91
+albucore==0.0.24
+lpips==0.1.4
+torchsde==0.2.6
+tomesd==0.1.3
+yarl==1.23.0
+typer==0.24.1
+triton==3.5.0
+cycler==0.12.1
+multidict==6.7.1
+OpenEXR==3.4.10
+soxr==1.0.0
+kiwisolver==1.5.0
+simsimd==6.5.16
+parso==0.8.6
+einops==0.8.2
+ftfy==6.3.1
+platformdirs==4.9.6
+groovy==0.1.2
+nvidia-nvtx-cu12==12.6.77
+sentry-sdk==2.58.0
+certifi==2026.2.25
+markdown-it-py==4.0.0
+uvicorn==0.38.0
+nvidia-cudnn-cu12==9.10.2.21
+natsort==8.4.0
+awscrt==0.31.2
+accelerate==1.13.0
+deepspeed==0.18.9
+matplotlib==3.10.8
+mdurl==0.1.2
+regex==2026.4.4
+nvidia-cuda-cupti-cu12==12.6.80
+shellingham==1.5.4
+joblib==1.5.3
+zipp==3.23.1
+ipython_pygments_lexers==1.1.1
+executing==2.2.1
+safetensors==0.7.0
+typing-inspection==0.4.2
+httpcore==1.0.9
+semantic-version==2.10.0
+wandb==0.26.0
+asttokens==3.0.1
+datasets==4.8.4
+decorator==5.2.1
+attrs==26.1.0
+six==1.17.0
+importlib_metadata==9.0.0
+py-cpuinfo==9.0.0
+requests==2.33.1
+xxhash==3.6.0
+pytorch-lightning==2.6.1
+fsspec==2026.2.0
+albumentations==2.0.8
+soundfile==0.13.1
+torchaudio==2.9.0
+torchmetrics==1.9.0
+nvidia-cusolver-cu12==11.7.1.2
+scipy==1.17.1
+jmespath==1.1.0
+threadpoolctl==3.6.0
+plyfile==1.1.2
+peft==0.19.1
+pycparser==3.0
+Markdown==3.10.2
+aiosignal==1.4.0
+pydantic==2.13.2
+decord2==3.3.0
+safehttpx==0.1.7
+scikit-image==0.26.0
+fonttools==4.62.1
+lazy-loader==0.5
+smmap==5.0.3
+starlette==1.0.0
+Werkzeug==3.1.8
+propcache==0.4.1
+tensorboard==2.20.0
+rich==15.0.0
+modelscope==1.36.3
+absl-py==2.4.0
+traitlets==5.14.3
+networkx==3.6.1
+hf-xet==1.4.3
+h5py==3.16.0
+PyYAML==6.0.3
+Jinja2==3.1.6
+nvidia-cusparse-cu12==12.5.4.2
+idna==3.11
+transformers==4.46.2
+httpx==0.28.1
+MarkupSafe==3.0.3
+tensorboard-data-server==0.7.2
+pooch==1.9.0
+torch==2.9.0+cu126
+soupsieve==2.8.3
+numba==0.65.0
+torchdiffeq==0.2.5
+jedi==0.19.2
+charset-normalizer==3.4.7
+annotated-doc==0.0.4
+gradio==6.12.0
+Pygments==2.20.0
+pandas==3.0.2
+stack-data==0.6.3
+contourpy==1.3.3
+tomlkit==0.14.0
+scikit-learn==1.8.0
+librosa==0.11.0
+nvidia-cusparselt-cu12==0.7.1
+pytz==2026.1.post1
+go-with-the-track==0.1.0
+yunchang==0.6.4
+onnxruntime==1.24.4
+ninja==1.13.0
+sentencepiece==0.2.1
+pexpect==4.9.0
+multiprocess==0.70.19
+omegaconf==2.3.0
+h11==0.16.0
+pydub==0.25.1
+nvidia-cufft-cu12==11.3.0.4
+xfuser==0.4.5
+tifffile==2026.3.3
+msgpack==1.1.2
+psutil==7.2.2
+ipython==9.12.0
+hf-gradio==0.4.1
+diffusers==0.33.1
+python-dateutil==2.9.0.post0
+orjson==3.11.8
+huggingface_hub==0.36.2
+nvidia-cuda-runtime-cu12==12.6.77
+beautifulsoup4==4.14.3
+ptyprocess==0.7.0
+protobuf==7.34.1
+GitPython==3.1.46
+sympy==1.14.0
+packaging==26.1
+aiohttp==3.13.5
+nvidia-nvshmem-cu12==3.3.20
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+gradio_client==2.4.1
+stringzilla==4.6.0
+flatbuffers==25.12.19
+filelock==3.28.0
+imageio-ffmpeg==0.6.0
+fastapi==0.136.0
+nvidia-nvjitlink-cu12==12.6.85
+s3transfer==0.16.0
+python-multipart==0.0.26
+DistVAE==0.0.0b5
+urllib3==2.6.3
+nvidia-nccl-cu12==2.27.5
+av==15.1.0
+pure_eval==0.2.3
+wheel==0.46.3
+dill==0.4.1
+flash_attn==2.8.3
+track4d-360==0.1.0
+opencv-python==4.11.0.86
+anyio==4.13.0
+pytest==9.0.3
+nvidia-cufile-cu12==1.11.1.6
+lightning-utilities==0.15.3
+pluggy==1.6.0
+grpcio==1.80.0
+cffi==2.0.0
+iniconfig==2.3.0
+videox-fun==1.0.1
+matplotlib-inline==0.2.1
+tokenizers==0.20.3
+pydantic_core==2.46.2
+aiohappyeyeballs==2.6.1
+typing_extensions==4.15.0
+timm==1.0.26
+pyarrow==23.0.1
+mpmath==1.3.0
+prompt_toolkit==3.0.52
+mediapy==1.2.6
+llvmlite==0.47.0
+opencv-python-headless==4.11.0.86
+hjson==3.1.0
+tqdm==4.67.3
+torchvision==0.24.0
+pyparsing==3.3.2
+nvidia-cublas-cu12==12.6.4.1
+botocore==1.42.91
+nanobind==2.9.2
+wcwidth==0.6.0
+gitdb==4.0.12
+frozenlist==1.8.0
+brotli==1.2.0
+trampoline==0.1.2
+pillow==12.2.0
+audioread==3.1.0
+lightning==2.6.1
+setuptools==82.0.1
+nvidia-curand-cu12==10.3.7.77

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/files/wandb-metadata.json

@@ -0,0 +1,235 @@
+{
+  "os": "Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-06-02T06:50:19.986166Z",
+  "args": [
+    "--model_id_with_origin_paths",
+    "Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth",
+    "--tokenizer_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl",
+    "--task",
+    "sft:v2v",
+    "--model_variant",
+    "vt2v",
+    "--v2v_concat_mode",
+    "width",
+    "--height",
+    "288",
+    "--width",
+    "512",
+    "--num_frames",
+    "49",
+    "--track4d360_data_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml",
+    "--steps_per_epoch",
+    "25000",
+    "--num_epochs",
+    "200",
+    "--learning_rate",
+    "2e-5",
+    "--gradient_accumulation_steps",
+    "1",
+    "--save_steps",
+    "500",
+    "--remove_prefix_in_ckpt",
+    "pipe.dit.",
+    "--trainable_models",
+    "dit.domain_embedding,dit.control_patch_embedding",
+    "--dit_finetune_modules",
+    "self_attn",
+    "--track_injection_mode",
+    "per_block",
+    "--track_injection_block_mode",
+    "concat_project",
+    "--track_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml",
+    "--train_sigma_shift",
+    "5.0",
+    "--use_query_frame_impulse_condition",
+    "--use_plucker_camera_control",
+    "--enable_v2v_plucker_camera_control",
+    "--use_plucker_raymap_tokens",
+    "--use_perlayer_camera_control",
+    "--camera_injection_mode",
+    "projector",
+    "--camera_plucker_encoder_mode",
+    "patchify",
+    "--use_dense_branch",
+    "--dense_in_channels",
+    "5",
+    "--dense_proj_dim",
+    "32",
+    "--dense_num_residual_blocks",
+    "0",
+    "--dense_alpha_track",
+    "1.0",
+    "--dense_conditioning_noise_max_level",
+    "0.15",
+    "--source_mask_boundary_depth_dropout_prob",
+    "0.5",
+    "--source_mask_boundary_kernel_size",
+    "5",
+    "--dense_warp_mode",
+    "hybrid_softsplat",
+    "--dense_warp_motion_threshold",
+    "0.005",
+    "--cond_drop_geo_prob",
+    "0.1",
+    "--cond_drop_track_prob",
+    "0.1",
+    "--cond_drop_dense_prob",
+    "0.0",
+    "--initial_step",
+    "2530",
+    "--output_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout",
+    "--dataset_num_workers",
+    "7",
+    "--dataloader_prefetch_factor",
+    "2",
+    "--drop_text_prob",
+    "0.1",
+    "--debug_latents_every",
+    "500",
+    "--allow_existing_checkpoints",
+    "--wandb_project",
+    "track4d-360",
+    "--wandb_name",
+    "ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp",
+    "--wandb_tags",
+    "ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug",
+    "--use_wandb",
+    "--vxf_init_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors",
+    "--use_gradient_checkpointing",
+    "--resume_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors",
+    "--max_train_steps",
+    "2560",
+    "--torch_profile_active_steps",
+    "5",
+    "--torch_profile_wait_steps",
+    "3",
+    "--torch_profile_warmup_steps",
+    "1",
+    "--torch_profile_repeat",
+    "1",
+    "--no_torch_profile_record_shapes",
+    "--no_torch_profile_memory"
+  ],
+  "program": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py",
+  "codePath": "scripts/wan2.1_fun/train_track4d360.py",
+  "codePathLocal": "scripts/wan2.1_fun/train_track4d360.py",
+  "git": {
+    "remote": "git@github.com:NIRVANALAN/track4d-360.git",
+    "commit": "b6681ed2127310d14bc5b93ae5f1d20df8b9e907"
+  },
+  "email": "yushi.lan@outlook.com",
+  "root": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout",
+  "host": "nid010552",
+  "executable": "/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12",
+  "cpu_count": 288,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA GH200 120GB",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "255795724288",
+      "used": "527630336"
+    }
+  },
+  "memory": {
+    "total": "919613407232"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-5728e766-d093-23d3-0fed-a94feb031fe3"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-098a504b-2d3b-7b84-0901-6fbce487448c"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-4752a23e-8a0b-30b8-6dca-8c3bedf9c98c"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-c54beba8-b9e0-3543-a467-81ae596f05f6"
+    }
+  ],
+  "cudaVersion": "12.7",
+  "slurm": {
+    "cluster_name": "gracehopper",
+    "conf": "/var/spool/slurmd/conf-cache/slurm.conf",
+    "cpu_bind": "quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_list": "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_type": "mask_cpu:",
+    "cpu_bind_verbose": "quiet",
+    "cpus_on_node": "288",
+    "distribution": "cyclic",
+    "gpus": "32",
+    "gpus_on_node": "4",
+    "gpus_per_node": "4",
+    "gtids": "0",
+    "job_account": "brics.u6ew",
+    "job_cpus_per_node": "288(x8)",
+    "job_end_time": "1780389885",
+    "job_gid": "1483804847",
+    "job_gpus": "0,1,2,3",
+    "job_id": "4982943",
+    "job_name": "14b_pair1x49_profile_ablate_bothres0_fsdp2",
+    "job_nodelist": "nid[010552-010553,010558,010561-010565]",
+    "job_num_nodes": "8",
+    "job_partition": "workq",
+    "job_qos": "normal",
+    "job_start_time": "1780382685",
+    "job_uid": "1483804847",
+    "job_user": "yushi.u6ew",
+    "jobid": "4982943",
+    "launch_node_ipaddr": "10.242.15.10",
+    "localid": "0",
+    "mpi_type": "pmi2",
+    "nnodes": "8",
+    "nodeid": "0",
+    "nodelist": "nid[010552-010553,010558,010561-010565]",
+    "nprocs": "8",
+    "ntasks": "8",
+    "ntasks_per_node": "1",
+    "oom_kill_step": "0",
+    "prio_process": "0",
+    "procid": "0",
+    "srun_comm_host": "10.242.15.10",
+    "srun_comm_port": "41219",
+    "step_gpus": "0,1,2,3",
+    "step_id": "0",
+    "step_launcher_port": "41219",
+    "step_nodelist": "nid[010552-010553,010558,010561-010565]",
+    "step_num_nodes": "8",
+    "step_num_tasks": "8",
+    "step_resv_ports": "26620-26621",
+    "step_tasks_per_node": "1(x8)",
+    "stepid": "0",
+    "submit_dir": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360",
+    "submit_host": "login40",
+    "task_pid": "248425",
+    "tasks_per_node": "1(x8)",
+    "topology_addr": "global.group7.nid010552",
+    "topology_addr_pattern": "switch.switch.node",
+    "umask": "0022"
+  },
+  "writerId": "4n8ybj8xil1hl68ckg9qyr4h3ok6yfl3"
+}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/timing/unit/WanVideoUnit_LatentNoiseAugmentation_s":4.6193774323910475e-05,"train/hbm/post_t5_allocated_gb":28.869243621826172,"train/track/injector/block_28/grad_norm":0.003465704619884491,"train/track/injector/block_03/grad_norm":0.01044765766710043,"train/track/injector/block_34/grad_norm":0.0024946231860667467,"train/track/injector/block_30/grad_norm":0.002777846995741129,"train/track/injector/block_16/weight_norm":71.75880432128906,"train/timing/unit/WanVideoUnit_TeaCache_s":1.9860548491124064e-05,"train/track/injector/block_31/grad_norm":0.0024459627456963062,"train/track/injector/block_16/grad_norm":0.00687984237447381,"train/track/injector/block_33/weight_norm":71.65996551513672,"train/track/injector/block_37/weight_norm":71.64706420898438,"_runtime":575,"train/track/injector/block_10/grad_norm":0.008368905633687973,"train/track/injector/block_07/weight_norm":71.78174591064453,"train/timing/unit/WanVideoUnit_NoiseInitializer_s":0.00017372827278450131,"train/track/injector/block_15/weight_norm":71.72611999511719,"train/track/injector/block_20/weight_norm":71.71338653564453,"_timestamp":1.780383580238531e+09,"train/track/injector/block_22/weight_norm":71.71588897705078,"train/timing/other_s":0.8354626465588808,"train/track/injector/block_25/weight_norm":71.73038482666016,"train/track/injector/block_09/weight_norm":71.7808609008789,"train/track/injector/block_19/weight_norm":71.72639465332031,"train/track/injector/block_23/weight_norm":71.7166519165039,"train/track/injector/block_26/weight_norm":71.73136901855469,"train/t5/on_cuda":-1,"train/timing/unit/WanVideoUnit_CameraControl_NVS_s":0.0032826303504407406,"train/track/injector/block_25/grad_norm":0.0038604442961513996,"train/track/injector/block_12/grad_norm":0.0076773264445364475,"train/track/injector/block_23/grad_norm":0.004788658581674099,"train/track/injector/block_00/weight_norm":71.61990356445312,"train/track/injector/block_24/grad_norm":0.004532807506620884,"train/track/injector/block_28/weight_norm":71.74323272705078,"train/track/injector/block_35/grad_norm":0.0023304815404117107,"train/hbm/post_t5_reserved_gb":28.94489097595215,"_step":2559,"train/track/injector/block_36/grad_norm":0.002490128856152296,"train/track/injector/block_14/weight_norm":71.75389099121094,"train/track/injector/block_19/grad_norm":0.005257687531411648,"train/track/adapter/weight_norm":16.34898567199707,"train/track/injector/block_22/grad_norm":0.0061228470876812935,"train/track/injector/weight_norm_max":71.8016357421875,"train/track/injector/block_21/grad_norm":0.004948248155415058,"train/track/injector/grad_norm_mean":0.006771211978048086,"train/track/injector/block_04/weight_norm":71.78126525878906,"train/track/injector/block_07/grad_norm":0.009389299899339676,"train/timing/forward_s":5.398297315929085,"train/track/injector/block_01/grad_norm":0.01250562071800232,"train/timing/unit/WanVideoUnit_WarpedAppearance_s":5.145258910488337e-05,"train/track/injector/block_33/grad_norm":0.0026537368539720774,"train/timing/data_wait_s":0.007866659667342901,"train/track/adapter/grad_norm":0.0046793692745268345,"train/track/injector/block_06/weight_norm":71.78472137451172,"train/timing/unit/WanVideoUnit_ShapeChecker_s":5.178374703973532e-05,"train/timing/unit/WanVideoUnit_TargetVideoEncoder_s":0.8000562191009521,"train/timing/unit/WanVideoUnit_PromptEmbedder_s":0.00022612593602389097,"train/track/injector/block_31/weight_norm":71.70204162597656,"train/track/injector/block_03/weight_norm":71.8016357421875,"train/track/injector/block_11/grad_norm":0.008123037405312061,"train/track/injector/block_01/weight_norm":71.73564910888672,"train/cond_dropout/drop_dense":0.0625,"train/track/injector/block_05/weight_norm":71.79096221923828,"train/loss":0.06599140167236328,"train/timing/backward_s":5.112877536565065,"train/track/injector/block_05/grad_norm":0.009117974899709225,"train/track/injector/block_27/grad_norm":0.0034159559290856123,"train/timing/unit/WanVideoUnit_V2VConcatenate_s":0.00012627961405087262,"train/track/injector/block_37/grad_norm":0.002511002356186509,"train/timing/dit_loss_fn_s":2.68336820602417,"train/track/injector/block_24/weight_norm":71.72334289550781,"train/track/injector/block_29/grad_norm":0.0035822265781462193,"train/track/injector/block_04/grad_norm":0.009663588367402554,"train/cond_dropout/drop_tracks":0.125,"train/track/injector/block_02/weight_norm":71.77011108398438,"train/timing/unit/WanVideoUnit_ModeDropoutAugmentation_s":2.7219219191465527e-05,"train/track/injector/block_34/weight_norm":71.64427947998047,"train/track/injector/block_02/grad_norm":0.010977485217154026,"train/track/injector/block_10/weight_norm":71.777587890625,"train/track/injector/block_11/weight_norm":71.75846099853516,"train/track/injector/block_39/grad_norm":0.010027051903307438,"train/timing/unit/WanVideoUnit_TrackPointCondition_s":0.02079019695520401,"train/track/injector/block_13/weight_norm":71.73658752441406,"train/track/injector/block_38/weight_norm":71.60935974121094,"train/track/injector/block_06/grad_norm":0.00888015516102314,"train/track/injector/block_13/grad_norm":0.007709243334829807,"train/hbm/allocated_gb":3.43538236618042,"train/timing/unit/WanVideoUnit_Vista4DStreams_s":5.778144986834377e-05,"train/track/injector/block_18/grad_norm":0.005994467996060848,"train/timing/reduction_s":0.15421424619853497,"train/track/injector/grad_norm_max":0.035497479140758514,"train/track/injector/block_18/weight_norm":71.7178955078125,"train/track/injector/block_30/weight_norm":71.70831298828125,"train/track/injector/weight_norm_mean":71.72107696533203,"train/track/injector/block_27/weight_norm":71.70165252685547,"_wandb":{"runtime":575},"train/timing/unit/WanVideoUnit_DenseTargetControl_s":0.26256734132766724,"train/track/injector/block_15/grad_norm":0.006465190555900335,"train/track/injector/block_38/grad_norm":0.003349488601088524,"train/track/injector/block_29/weight_norm":71.70823669433594,"train/hbm/reserved_gb":39.54901123046875,"train/track/injector/block_32/weight_norm":71.66661071777344,"train/track/injector/block_09/grad_norm":0.008560985326766968,"train/track/injector/block_39/weight_norm":71.61332702636719,"train/track/injector/block_35/weight_norm":71.6831283569336,"train/track/injector/block_14/grad_norm":0.007657380774617195,"train/track/injector/block_00/grad_norm":0.035497479140758514,"train/track/injector/block_21/weight_norm":71.712646484375,"train/track/injector/block_08/grad_norm":0.009059657342731953,"train/track/injector/block_17/weight_norm":71.72586822509766,"train/epoch":0,"train/timing/microsteps":1,"train/timing/unit/WanVideoUnit_UnifiedSequenceParallel_s":2.6808316761162132e-05,"train/timing/unit/WanVideoUnit_CfgMerger_s":1.3150616723578423e-05,"train/track/injector/block_17/grad_norm":0.00607278011739254,"train/track/injector/block_26/grad_norm":0.0038510886952281,"train/track/injector/block_08/weight_norm":71.77731323242188,"train/track/injector/block_20/grad_norm":0.004393440671265125,"train/timing/step_total_s":11.538062416017056,"train/track/injector/block_12/weight_norm":71.75679016113281,"train/track/injector/block_36/weight_norm":71.64958190917969,"train/timing/optimizer_s":0.03721067076548934,"train/timing/unit/WanVideoUnit_InputVideoEmbedder_XYZCondition_s":1.5892174243927002,"train/track/injector/block_32/grad_norm":0.0025009731762111187}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/logs/debug-internal.log

@@ -0,0 +1,95 @@
+{"time":"2026-06-02T06:50:20.584631685Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T06:50:20.58696336Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T06:50:20.969725714Z","level":"INFO","msg":"stream: created new stream","id":"3241szoo"}
+{"time":"2026-06-02T06:50:20.96994734Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T06:50:20.970735157Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T06:50:20.9707597Z","level":"INFO","msg":"writer: started","stream_id":"3241szoo"}
+{"time":"2026-06-02T06:50:20.971093546Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T06:50:22.923197858Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T06:50:23.133364725Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:30.303531233Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T06:50:30.305530822Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":47}
+{"time":"2026-06-02T06:50:37.923464724Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":115,"uploaded_len":2}
+{"time":"2026-06-02T06:50:38.126829678Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:52.923591564Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T06:50:53.109219757Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:07.924029832Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-06-02T06:51:08.114725101Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:22.92352313Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-06-02T06:51:23.119009653Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:37.924474754Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-06-02T06:51:38.141711785Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:52.923701659Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":116,"console_lines":1}
+{"time":"2026-06-02T06:51:53.111102047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:07.924313952Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":117,"console_lines":3}
+{"time":"2026-06-02T06:52:08.119941045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:22.924342752Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:52:23.14398364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:37.924171237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":2,"events_offset":16,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:52:38.127651722Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:52.923475326Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:52:53.111875741Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:07.924090817Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:08.123142555Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:22.924082144Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":22,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:23.155915012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:37.924127613Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:38.132051519Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:52.924552016Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:53:53.117190814Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:07.923825475Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:54:08.143248291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:22.924253036Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T06:54:23.162918899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:37.924269819Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T06:54:38.178293682Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:52.92343901Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-06-02T06:54:53.126764907Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:07.924320537Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-06-02T06:55:08.132013064Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:22.924083496Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-06-02T06:55:23.183084706Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:37.923701069Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
+{"time":"2026-06-02T06:55:38.176471756Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:52.924750892Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:55:53.115223076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:07.923910025Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:08.170806645Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:22.924071465Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:23.147054865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:37.923990093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:38.174203624Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:52.924661661Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:56:53.139214445Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:07.924453061Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:08.169695656Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:22.924163918Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":54,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:23.146405546Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:37.924201145Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:38.109290006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:52.925187457Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:57:53.15045367Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:07.924653715Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":60,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:08.180545128Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:22.923699153Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:23.120052374Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:37.923835946Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:38.182881949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:52.923692065Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:58:53.140186235Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:07.924480574Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":2,"events_offset":68,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:08.135037406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:22.923975937Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":70,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:23.116919373Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:37.924640345Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:38.187698331Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:52.92428249Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":115,"console_lines":1}
+{"time":"2026-06-02T06:59:53.247046902Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:57.467547914Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T06:59:57.467952574Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":120,"console_lines":92,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T06:59:57.66002893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:57.660797388Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T06:59:57.665684932Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T06:59:57.665718499Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T06:59:57.666134903Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T06:59:57.66616015Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/logs/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 06:50:19,995 INFO    MainThread:248506 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_setup.py:_flush():81] Configure stats pid to 248506
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/logs/debug.log
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/logs/debug-internal.log
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:init():848] calling init triggers
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 0, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 06:50:19,996 INFO    MainThread:248506 [wandb_init.py:init():896] starting backend
+2026-06-02 06:50:20,574 INFO    MainThread:248506 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 06:50:20,582 INFO    MainThread:248506 [wandb_init.py:init():919] backend started and connected
+2026-06-02 06:50:20,584 INFO    MainThread:248506 [wandb_init.py:init():989] updated telemetry
+2026-06-02 06:50:20,629 INFO    MainThread:248506 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 06:50:21,612 INFO    MainThread:248506 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 06:50:22,909 INFO    MainThread:248506 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 06:50:22,909 INFO    MainThread:248506 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 06:50:22,909 INFO    MainThread:248506 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 06:50:22,910 INFO    MainThread:248506 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 06:50:22,916 INFO    MainThread:248506 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 06:59:56,752 INFO    MainThread:248506 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/3241szoo
+2026-06-02 06:59:56,772 INFO    MainThread:248506 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 06:59:56,773 INFO    MainThread:248506 [wandb_run.py:_restore():2490] restore
+2026-06-02 06:59:56,773 INFO    MainThread:248506 [wandb_run.py:_restore():2496] restore done
+2026-06-02 06:59:57,664 INFO    MainThread:248506 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_bothres0_cond_dropout/wandb/run-20260602_065019-3241szoo/run-3241szoo.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93acd73fe6c3a00e2250c4b09d384b5416f96bc641620d896dd946bfffcd54b2
+size 686914

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+#SBATCH --job-name=14b_8node_288x512_pair1x49_next_eb32_lr2e5
+#SBATCH --nodes=8
+#SBATCH --gpus=32
+#SBATCH --gpus-per-node=4
+#SBATCH --ntasks-per-node=1
+#SBATCH --time=24:00:00
+#SBATCH --output=out/exp_%j_%x.out
+
+# Track4D-360 14B FSDP 288x512 next-stage finetune.
+#
+# Main changes from the 288p backendmix OOM-debug launcher:
+#   - new DATA_TAG / output directory;
+#   - warm-start from the latest existing 288p backendmix checkpoint;
+#   - remove most static datasets from DATA_CONFIG;
+#   - add ReCamMaster CoWTracker + square512 source49 + compact pair1x49
+#     VGGT-Omega labels, q1-only by dataset construction.
+#
+# Submit from repo root on Isambard:
+#   sbatch bash_scripts/track4d_360/ismb/14b/sbatch/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh
+
+export NNODES=8
+export MODEL_SIZE=14b
+export CONDA_ENV="${CONDA_ENV:-th29py312}"
+export HEIGHT=288
+export WIDTH=512
+export NUM_FRAMES=49
+export main_process_port="${main_process_port:-29595}"
+
+export WAN_T2V14B_DIR="${WAN_T2V14B_DIR:-${SLURM_SUBMIT_DIR}/weights/Wan2.1-T2V-14B}"
+export VXF_INIT="${VXF_INIT:-${WAN_T2V14B_DIR}/diffusion_pytorch_model*.safetensors}"
+export MODEL_ID_PREFIX="${MODEL_ID_PREFIX:-Wan-AI/Wan2.1-T2V-14B}"
+export TOKENIZER_PATH="${TOKENIZER_PATH:-${WAN_T2V14B_DIR}/google/umt5-xxl}"
+export T5_PATH="${T5_PATH:-${WAN_T2V14B_DIR}/models_t5_umt5-xxl-enc-bf16.pth}"
+export VAE_PATH="${VAE_PATH:-${WAN_T2V14B_DIR}/Wan2.1_VAE.pth}"
+
+# Prefer the current 288p backendmix run as the next-stage warm start. Fall
+# back to the 384p backendmix/raymap-token parents only when no 288p checkpoint exists.
+export BACKENDMIX_288_PARENT_DIR="${BACKENDMIX_288_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout}"
+export BACKENDMIX_384_PARENT_DIR="${BACKENDMIX_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout}"
+export RAYMAP_384_PARENT_DIR="${RAYMAP_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout}"
+# Warm-start from the 288p backendmix run's step-9000 checkpoint — the requested
+# next-stage starting point. (FSDP1/FSDP2 read it identically: warm-start loads
+# the safetensors BEFORE accelerator.prepare, on the unwrapped model, so it is
+# distributed-backend-independent.) Override WARM_START=... to pick another;
+# falls back to latest backendmix / 384p parents if step-9000 is absent.
+if [ -z "${WARM_START:-}" ]; then
+    if [ -f "${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors" ]; then
+        export WARM_START="${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors"
+    else
+        latest="$(find "${BACKENDMIX_288_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        if [ -z "${latest}" ]; then
+            latest="$(find "${BACKENDMIX_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        if [ -z "${latest}" ]; then
+            latest="$(find "${RAYMAP_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        export WARM_START="${latest:-${RAYMAP_384_PARENT_DIR}/step-3500.safetensors}"
+    fi
+fi
+
+if ! compgen -G "${VXF_INIT}" >/dev/null; then
+    echo "ERROR: VXF_INIT not found: ${VXF_INIT}" >&2
+    exit 1
+fi
+[ -d "${TOKENIZER_PATH}" ] || { echo "ERROR: TOKENIZER_PATH not found: ${TOKENIZER_PATH}" >&2; exit 1; }
+[ -f "${T5_PATH}" ] || { echo "ERROR: T5_PATH not found: ${T5_PATH}" >&2; exit 1; }
+[ -f "${VAE_PATH}" ] || { echo "ERROR: VAE_PATH not found: ${VAE_PATH}" >&2; exit 1; }
+
+export TARGET_EFFECTIVE_BATCH=32
+export gradient_accumulation_steps=1
+export learning_rate="${learning_rate:-2e-5}"
+export save_steps="${save_steps:-500}"
+export dataset_num_workers="${dataset_num_workers:-7}"
+export TRACK4D360_STOP_AFTER_NEXT_SAVE="${TRACK4D360_STOP_AFTER_NEXT_SAVE:-0}"
+
+export dit_finetune_modules="${dit_finetune_modules:-self_attn}"
+export DIT_FINETUNE_TAG="${DIT_FINETUNE_TAG:-attn}"
+export use_plucker_raymap_tokens=1
+export train_plucker_camera_control=0
+export PLUCKER_WANDB_TAG="${PLUCKER_WANDB_TAG:-plucker_raymap_tokens}"
+
+export DATA_CONFIG="${DATA_CONFIG:-${SLURM_SUBMIT_DIR}/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml}"
+export DATA_TAG="${DATA_TAG:-mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512}"
+
+export RECAMMASTER_SOURCE49_ROOT="${RECAMMASTER_SOURCE49_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}"
+export RECAMMASTER_PAIR1X49_LABEL_ROOT="${RECAMMASTER_PAIR1X49_LABEL_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}"
+export RECAMMASTER_COW_ROOT="${RECAMMASTER_COW_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}"
+export TRACK4D360_RECAM_TRACK_SAMPLE_MODE="${TRACK4D360_RECAM_TRACK_SAMPLE_MODE:-grid}"
+export TRACK4D360_RECAM_TRACK_STRIDE="${TRACK4D360_RECAM_TRACK_STRIDE:-2}"
+export TRACK4D360_RECAM_TRACK_SAMPLES="${TRACK4D360_RECAM_TRACK_SAMPLES:-0}"
+# Pair1x49 labels currently cover only the priority block: first 750 numeric
+# train scenes inside each K/focal group (e.g. train/f18_aperture10/scene000..).
+export TRACK4D360_RECAM_SCENE_OFFSET_PER_K="${TRACK4D360_RECAM_SCENE_OFFSET_PER_K:-0}"
+export TRACK4D360_RECAM_SCENE_LIMIT_PER_K="${TRACK4D360_RECAM_SCENE_LIMIT_PER_K:-750}"
+
+# Fixed post-compaction track identity count — MUST be set (the 24h/10k-step
+# backendmix run had this at 65536; this launcher previously omitted it). The
+# ReCamMaster CoW tracks are dense (grid stride 2, samples=0 = uncapped), so
+# without a cap N varies per sample and a heavy draw spikes the track adapter's
+# proj_down activation (B,N,T,hidden) -> the observed CUDA OOM at
+# conv3d_track_adapter.py proj_down (1.16 GiB alloc, step ~770). Fixing N also
+# pads small draws to a constant shape, eliminating per-step fragmentation. Lower
+# to 32768 for extra headroom on contended nodes.
+export TRACK4D360_TRACK_IDENTITY_FIXED_N="${TRACK4D360_TRACK_IDENTITY_FIXED_N:-65536}"
+
+export TRACK4D360_SPARSE_TRACK_PROB="${TRACK4D360_SPARSE_TRACK_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_STRIDE="${TRACK4D360_SPARSE_TRACK_STRIDE:-2|4|8}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB="${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_SIZE="${TRACK4D360_SPARSE_TRACK_BLOCK_SIZE:-64}"
+export TRACK4D360_MULTI_QUERY_PROB="${TRACK4D360_MULTI_QUERY_PROB:-1.0}"
+export TRACK4D360_MULTI_QUERY_CHOICES="${TRACK4D360_MULTI_QUERY_CHOICES:-1|2|3|4}"
+export TRACK4D360_GEOMETRY_NOISE_PROB="${TRACK4D360_GEOMETRY_NOISE_PROB:-1.0}"
+export TRACK4D360_TRACK_OFFSET_STD="${TRACK4D360_TRACK_OFFSET_STD:-0.015}"
+export TRACK4D360_TRACK_TEMPORAL_JITTER_STD="${TRACK4D360_TRACK_TEMPORAL_JITTER_STD:-0.006}"
+export TRACK4D360_TRACK_XY_NOISE_RATIO="${TRACK4D360_TRACK_XY_NOISE_RATIO:-0.25}"
+export TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO="${TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO:-0.5}"
+export TRACK4D360_DEPTH_OFFSET_STD="${TRACK4D360_DEPTH_OFFSET_STD:-0.006}"
+export TRACK4D360_DEPTH_SCALE_JITTER="${TRACK4D360_DEPTH_SCALE_JITTER:-0.01}"
+export TRACK4D360_SOURCE_RGB_NOISE_STD="${TRACK4D360_SOURCE_RGB_NOISE_STD:-0.01}"
+export TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD="${TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD:-0.02}"
+export TRACK4D360_DYNAMIC_MASK_AUG_PROB="${TRACK4D360_DYNAMIC_MASK_AUG_PROB:-0.5}"
+export TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL="${TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL:-5}"
+export TRACK4D360_DYNAMIC_MASK_FLIP_PROB="${TRACK4D360_DYNAMIC_MASK_FLIP_PROB:-0.002}"
+
+export dense_warp_mode="${dense_warp_mode:-hybrid_softsplat}"
+export dense_warp_motion_threshold="${dense_warp_motion_threshold:-0.005}"
+export dense_conditioning_noise_max_level="${dense_conditioning_noise_max_level:-0.15}"
+export source_mask_boundary_depth_dropout_prob="${source_mask_boundary_depth_dropout_prob:-0.5}"
+export source_mask_boundary_kernel_size="${source_mask_boundary_kernel_size:-5}"
+export freeze_track4d_condition_modules="${freeze_track4d_condition_modules:-0}"
+
+export use_gradient_checkpointing=1
+export use_gradient_checkpointing_offload="${use_gradient_checkpointing_offload:-0}"
+export VXF_OFFLOAD_T5_TO_CPU="${VXF_OFFLOAD_T5_TO_CPU:-1}"
+export TRACK4D360_MEM_PROBE="${TRACK4D360_MEM_PROBE:-0}"
+export TRACK4D360_ADAPTER_MEM_PROFILE="${TRACK4D360_ADAPTER_MEM_PROFILE:-0}"
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-${PYTORCH_ALLOC_CONF:-expandable_segments:True}}"
+export PYTORCH_ALLOC_CONF="${PYTORCH_ALLOC_CONF:-${PYTORCH_CUDA_ALLOC_CONF}}"
+export TRACK4D360_DENSE_HYBRID_CACHE_FRAMES="${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES:-7}"
+export TRACK4D360_DENSE_HYBRID_TGT_CHUNK="${TRACK4D360_DENSE_HYBRID_TGT_CHUNK:-2}"
+
+export NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-0}"
+export NCCL_P2P_LEVEL="${NCCL_P2P_LEVEL:-NVL}"
+export NCCL_NVLS_ENABLE=0
+export NCCL_NSOCKS_PERTHREAD=8
+export NCCL_SOCKET_NTHREADS=8
+export NCCL_MAX_NCHANNELS="${NCCL_MAX_NCHANNELS:-32}"
+export NCCL_BUFFSIZE="${NCCL_BUFFSIZE:-8388608}"
+export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
+
+export TRACK4D360_CAPTION_OUTPUT_ROOT="${TRACK4D360_CAPTION_OUTPUT_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output}"
+export SYN4D_CAPTION_DIR="${SYN4D_CAPTION_DIR:-${TRACK4D360_CAPTION_OUTPUT_ROOT}/Syn4D}"
+
+echo "[raymap-tokens 14B pair1x49 next] T2V init: ${VXF_INIT}"
+echo "[raymap-tokens 14B pair1x49 next] warm start: ${WARM_START}"
+echo "[raymap-tokens 14B pair1x49 next] data: ${DATA_CONFIG}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam source49: ${RECAMMASTER_SOURCE49_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam pair1x49 labels: ${RECAMMASTER_PAIR1X49_LABEL_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam CoW: ${RECAMMASTER_COW_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam q1 track sampling: mode=${TRACK4D360_RECAM_TRACK_SAMPLE_MODE}, stride=${TRACK4D360_RECAM_TRACK_STRIDE}, samples=${TRACK4D360_RECAM_TRACK_SAMPLES}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam scene window per K: offset=${TRACK4D360_RECAM_SCENE_OFFSET_PER_K}, limit=${TRACK4D360_RECAM_SCENE_LIMIT_PER_K}"
+echo "[raymap-tokens 14B pair1x49 next] q policy: ReCam q1-only; synthetic choices=${TRACK4D360_MULTI_QUERY_CHOICES}"
+echo "[raymap-tokens 14B pair1x49 next] sparse synthetic tracks: stride_choices=${TRACK4D360_SPARSE_TRACK_STRIDE}, block_jitter_prob=${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB}"
+echo "[raymap-tokens 14B pair1x49 next] trainables: self_attn + raymap Ecam/camera_projector + active track/dense condition modules"
+echo "[raymap-tokens 14B pair1x49 next] freeze track/dense condition modules: ${freeze_track4d_condition_modules}"
+echo "[raymap-tokens 14B pair1x49 next] batch: effective=32, grad_accum=1, lr=${learning_rate}, save_steps=${save_steps}"
+echo "[raymap-tokens 14B pair1x49 next] memory: grad_ckpt_offload=${use_gradient_checkpointing_offload}, hybrid_cache_frames=${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES}, hybrid_tgt_chunk=${TRACK4D360_DENSE_HYBRID_TGT_CHUNK}, cuda_alloc_conf=${PYTORCH_CUDA_ALLOC_CONF}"
+
+source "${SLURM_SUBMIT_DIR}/bash_scripts/track4d_360/ismb/sbatch_common.sh"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml

@@ -0,0 +1,77 @@
+# Next-stage 288x512 backendmix finetune with cleaned ReCamMaster pair1x49 labels.
+#
+# Mix: 35% ReCamMaster pair1x49, 35% Syn4D, 5% Kubric49, 5% Kubric81,
+# and 20% legacy static datasets. The static 20% keeps the previous backendmix
+# relative weights: DL3DV:RE10K:MVS-Synth:TartanAir = 65000:7500:12500:25000.
+# ReCamMaster uses compact square512 source49 + pair1x49 VGGT-Omega scale
+# labels plus CoWTracker source tracks and is q1-only by construction.
+#
+# sbatch_common.sh still sanity-checks for the old MVS-Synth guard string:
+# exclude_sequence_ids=['0000']. This config has no MVS-Synth entry.
+
+num_views: 49
+resolution: [512, 288]
+
+syn4d_root: ${oc.env:SYN4D_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/Syn4D}
+syn4d_caption_dir: ${oc.env:SYN4D_CAPTION_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/Syn4D}
+syn4d_valid_sequences: configs/syn4d_valid_sequences.txt
+dl3dv_root: ${oc.env:DL3DV_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/dl3dv_tar}
+re10k_root: ${oc.env:RE10K_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_re10k}
+mvs_synth_root: ${oc.env:MVS_SYNTH_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_mvs_synth}
+tartanair_root: ${oc.env:TARTANAIR_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_tartanair}
+kubric49_root: ${oc.env:KUBRIC49_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_49frames}
+kubric81_root: ${oc.env:KUBRIC81_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_81frames}
+kubric49_caption_cache_dir: ${oc.env:KUBRIC49_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_49}
+kubric81_caption_cache_dir: ${oc.env:KUBRIC81_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_81}
+recammaster_caption_cache_dir: ${oc.env:RECAMMASTER_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/recammaster}
+recammaster_root: ${oc.env:RECAMMASTER_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset}
+multicam_fg_mask_root: ${oc.env:MULTICAM_FG_MASK_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_SAM3_FG_81f_sources_person_human_288x512}
+recammaster_source49_root: ${oc.env:RECAMMASTER_SOURCE49_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}
+recammaster_pair1x49_label_root: ${oc.env:RECAMMASTER_PAIR1X49_LABEL_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}
+recammaster_cow_root: ${oc.env:RECAMMASTER_COW_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}
+recammaster_track_sample_mode: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLE_MODE,grid}
+recammaster_track_stride: ${oc.env:TRACK4D360_RECAM_TRACK_STRIDE,2}
+recammaster_track_samples: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLES,0}
+recammaster_scene_offset_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_OFFSET_PER_K,0}
+recammaster_scene_limit_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_LIMIT_PER_K,750}
+
+sparse_track_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_PROB,1.0}
+sparse_track_stride: ${oc.env:TRACK4D360_SPARSE_TRACK_STRIDE,2|4|8}
+sparse_track_query_stride_mode: ${oc.env:TRACK4D360_SPARSE_TRACK_QUERY_STRIDE_MODE,query_count_floor}
+sparse_track_block_jitter_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB,1.0}
+sparse_track_block_size: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_SIZE,64}
+multi_query_prob: ${oc.env:TRACK4D360_MULTI_QUERY_PROB,1.0}
+multi_query_choices: ${oc.env:TRACK4D360_MULTI_QUERY_CHOICES,1|2|3|4}
+geometry_noise_prob: ${oc.env:TRACK4D360_GEOMETRY_NOISE_PROB,1.0}
+track_offset_std: ${oc.env:TRACK4D360_TRACK_OFFSET_STD,0.015}
+track_temporal_jitter_std: ${oc.env:TRACK4D360_TRACK_TEMPORAL_JITTER_STD,0.006}
+track_xy_noise_ratio: ${oc.env:TRACK4D360_TRACK_XY_NOISE_RATIO,0.25}
+track_frame_depth_jitter_ratio: ${oc.env:TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO,0.5}
+depth_offset_std: ${oc.env:TRACK4D360_DEPTH_OFFSET_STD,0.006}
+depth_scale_jitter: ${oc.env:TRACK4D360_DEPTH_SCALE_JITTER,0.01}
+source_rgb_noise_std: ${oc.env:TRACK4D360_SOURCE_RGB_NOISE_STD,0.01}
+source_rgb_frame_jitter_std: ${oc.env:TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD,0.02}
+dynamic_mask_aug_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_AUG_PROB,0.5}
+dynamic_mask_morph_kernel: ${oc.env:TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL,5}
+dynamic_mask_flip_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_FLIP_PROB,0.002}
+
+train_datasets:
+  # 35% ReCamMaster CoWTracker + square512 source49 + compact pair1x49 VGGT labels.
+  # This dataset is q1 only: it always emits track_query_indices=[0]. The
+  # default track density is grid stride 2 to avoid lifting every CoW pixel
+  # during training; override TRACK4D360_RECAM_TRACK_SAMPLE_MODE=all for dense.
+  - "87500 @ ReCamMasterCowVGGT_NVS(ROOT='${recammaster_root}', vggt_root='${recammaster_source49_root}', source_vggt_root='${recammaster_source49_root}', pair1x49_label_root='${recammaster_pair1x49_label_root}', geometry_source='pair1x49_label_scale_only', cow_root='${recammaster_cow_root}', fg_mask_root='${multicam_fg_mask_root}', caption_cache_dir='${recammaster_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, vggt_post_crop_mode='cow_aspect', rgb_post_crop_mode='cow_aspect', track_sample_mode='${recammaster_track_sample_mode}', track_stride=${recammaster_track_stride}, track_samples=${recammaster_track_samples}, scene_offset_per_k=${recammaster_scene_offset_per_k}, scene_limit_per_k=${recammaster_scene_limit_per_k})"
+  # 35% Syn4D dynamic multi-camera mesh-tracked 3D.
+  - "87500 @ GeometryRobustnessWrapper_NVS(Syn4D_Multi_NVS(ROOT='${syn4d_root}', caption_dir='${syn4d_caption_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, track_spatial_stride=1, exclude_scenes=['flying_group'], valid_sequences_file='${syn4d_valid_sequences}', use_mp4_rgb=True, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 49-frame dynamic data.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric49_root}', caption_cache_dir='${kubric49_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 81-frame dynamic data sampled to 49.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric81_root}', caption_cache_dir='${kubric81_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 11.82% DL3DV (real, DA3 depth). Static total is 20%; this keeps the old static relative weight.
+  - "29550 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${dl3dv_root}', dataset_type='dl3dv', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=29550, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=0.5, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, use_da3_depth=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 1.38% RE10K (real, DA3 depth).
+  - "3450 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${re10k_root}', dataset_type='re10k', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=3450, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 2.28% MVS-Synth (synthetic, GT depth; scene 0000 held out).
+  - "5700 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${mvs_synth_root}', dataset_type='mvs_synth', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=5700, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, exclude_sequence_ids=['0000']), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 4.52% TartanAir (static forward-moving scenes).
+  - "11300 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${tartanair_root}', dataset_type='tartanair', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=11300, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=1.0, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/torch_profiler/rank0.1780383283598487142.pt.trace.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4fa158ed2e4c92bf08152ca349aa5998f7b6572defa59490d8a8c6d1b6bf8e2
+size 4743420315

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/debug-internal.log

@@ -0,0 +1,97 @@
+{"time":"2026-06-02T06:50:13.570807992Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T06:50:13.573515504Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T06:50:14.064306628Z","level":"INFO","msg":"stream: created new stream","id":"ixxjov67"}
+{"time":"2026-06-02T06:50:14.064412769Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T06:50:14.065276714Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T06:50:14.065282794Z","level":"INFO","msg":"writer: started","stream_id":"ixxjov67"}
+{"time":"2026-06-02T06:50:14.06566832Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T06:50:15.631914228Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T06:50:16.106065701Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:23.169926895Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T06:50:23.18417797Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":175}
+{"time":"2026-06-02T06:50:30.632687583Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":126,"uploaded_len":2}
+{"time":"2026-06-02T06:50:31.057643889Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:45.633956602Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T06:50:45.995092564Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:00.632732136Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-06-02T06:51:01.013164986Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:15.632693827Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-06-02T06:51:15.973083783Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:30.632614271Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-06-02T06:51:30.902457359Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:45.632638154Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":127,"console_lines":3}
+{"time":"2026-06-02T06:51:46.012081416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:00.632782948Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:01.245405887Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:15.633245252Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:16.031118944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:30.632460243Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":16,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:31.064836975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:45.632586315Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:45.920833256Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:00.632560879Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:01.114903023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:15.633256291Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":22,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:16.048155401Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:30.633585048Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:31.116840996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:45.632811032Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:45.995726219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:00.632163343Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":130,"console_lines":1}
+{"time":"2026-06-02T06:54:01.04412747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:15.632919183Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T06:54:16.033687178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:30.632581426Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T06:54:31.115164341Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:45.631988187Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-06-02T06:54:46.16067707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:00.632478441Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-06-02T06:55:01.137467668Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:15.634021135Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-06-02T06:55:16.528050934Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:30.633071312Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:55:30.990818571Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:45.632948623Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:55:45.947339192Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:00.63331661Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:01.090740183Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:15.633357213Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:16.100888881Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:30.633508896Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:31.248935786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:45.633656982Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:46.023339526Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:00.633242778Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:01.000390992Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:15.632535397Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":2,"events_offset":54,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:16.015127404Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:30.633130173Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:30.99356342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:45.633329584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:46.07541001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:00.632386559Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:01.18224948Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:15.632888861Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":62,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:16.030413718Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:30.633023024Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:31.068636213Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:45.633331848Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:46.039435816Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:00.632360803Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":68,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:59:01.039092307Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:15.632705468Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-06-02T06:59:16.085584263Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:30.632579618Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-06-02T06:59:31.365655224Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:45.633048671Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":2,"events_offset":74,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:59:46.003907075Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T07:00:00.633135873Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":76,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T07:00:01.363370609Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T07:00:13.67931977Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T07:00:13.679909675Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":131,"console_lines":103,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T07:00:14.181094862Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T07:00:14.18215301Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T07:00:14.188249365Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T07:00:14.188298388Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T07:00:14.188708233Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T07:00:14.188737768Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 06:50:12,052 INFO    MainThread:152969 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 06:50:12,053 INFO    MainThread:152969 [wandb_setup.py:_flush():81] Configure stats pid to 152969
+2026-06-02 06:50:12,053 INFO    MainThread:152969 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 06:50:12,053 INFO    MainThread:152969 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/logs/debug.log
+2026-06-02 06:50:12,054 INFO    MainThread:152969 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/logs/debug-internal.log
+2026-06-02 06:50:12,054 INFO    MainThread:152969 [wandb_init.py:init():848] calling init triggers
+2026-06-02 06:50:12,060 INFO    MainThread:152969 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 2, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 06:50:12,060 INFO    MainThread:152969 [wandb_init.py:init():896] starting backend
+2026-06-02 06:50:12,777 INFO    MainThread:152969 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 06:50:13,568 INFO    MainThread:152969 [wandb_init.py:init():919] backend started and connected
+2026-06-02 06:50:13,571 INFO    MainThread:152969 [wandb_init.py:init():989] updated telemetry
+2026-06-02 06:50:13,661 INFO    MainThread:152969 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 06:50:14,653 INFO    MainThread:152969 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 06:50:15,631 INFO    MainThread:152969 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 07:00:12,998 INFO    MainThread:152969 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/ixxjov67
+2026-06-02 07:00:13,001 INFO    MainThread:152969 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 07:00:13,002 INFO    MainThread:152969 [wandb_run.py:_restore():2490] restore
+2026-06-02 07:00:13,002 INFO    MainThread:152969 [wandb_run.py:_restore():2496] restore done
+2026-06-02 07:00:14,187 INFO    MainThread:152969 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/files/config.yaml

@@ -0,0 +1,700 @@
+_wandb:
+    value:
+        cli_version: 0.26.0
+        e:
+            1m9b4etgqyfvjqvydz51tku5m7qgycq3:
+                args:
+                    - --model_id_with_origin_paths
+                    - Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+                    - --tokenizer_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+                    - --task
+                    - sft:v2v
+                    - --model_variant
+                    - vt2v
+                    - --v2v_concat_mode
+                    - width
+                    - --height
+                    - "288"
+                    - --width
+                    - "512"
+                    - --num_frames
+                    - "49"
+                    - --track4d360_data_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+                    - --steps_per_epoch
+                    - "25000"
+                    - --num_epochs
+                    - "200"
+                    - --learning_rate
+                    - "2e-5"
+                    - --gradient_accumulation_steps
+                    - "1"
+                    - --save_steps
+                    - "500"
+                    - --remove_prefix_in_ckpt
+                    - pipe.dit.
+                    - --trainable_models
+                    - dit.domain_embedding,dit.control_patch_embedding
+                    - --dit_finetune_modules
+                    - self_attn
+                    - --track_injection_mode
+                    - per_block
+                    - --track_injection_block_mode
+                    - concat_project
+                    - --track_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml
+                    - --train_sigma_shift
+                    - "5.0"
+                    - --use_query_frame_impulse_condition
+                    - --use_plucker_camera_control
+                    - --enable_v2v_plucker_camera_control
+                    - --use_plucker_raymap_tokens
+                    - --use_perlayer_camera_control
+                    - --camera_injection_mode
+                    - projector
+                    - --camera_plucker_encoder_mode
+                    - patchify
+                    - --use_dense_branch
+                    - --dense_in_channels
+                    - "5"
+                    - --dense_proj_dim
+                    - "32"
+                    - --dense_num_residual_blocks
+                    - "2"
+                    - --dense_alpha_track
+                    - "1.0"
+                    - --dense_conditioning_noise_max_level
+                    - "0.15"
+                    - --source_mask_boundary_depth_dropout_prob
+                    - "0.5"
+                    - --source_mask_boundary_kernel_size
+                    - "5"
+                    - --dense_warp_mode
+                    - hybrid_softsplat
+                    - --dense_warp_motion_threshold
+                    - "0.005"
+                    - --cond_drop_geo_prob
+                    - "0.1"
+                    - --cond_drop_track_prob
+                    - "0.1"
+                    - --cond_drop_dense_prob
+                    - "0.0"
+                    - --initial_step
+                    - "2530"
+                    - --output_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout
+                    - --dataset_num_workers
+                    - "7"
+                    - --dataloader_prefetch_factor
+                    - "2"
+                    - --drop_text_prob
+                    - "0.1"
+                    - --debug_latents_every
+                    - "500"
+                    - --allow_existing_checkpoints
+                    - --wandb_project
+                    - track4d-360
+                    - --wandb_name
+                    - ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp
+                    - --wandb_tags
+                    - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+                    - --use_wandb
+                    - --vxf_init_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+                    - --use_gradient_checkpointing
+                    - --resume_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+                    - --max_train_steps
+                    - "2560"
+                    - --torch_profile_active_steps
+                    - "5"
+                    - --torch_profile_wait_steps
+                    - "3"
+                    - --torch_profile_warmup_steps
+                    - "1"
+                    - --torch_profile_repeat
+                    - "1"
+                    - --no_torch_profile_record_shapes
+                    - --no_torch_profile_memory
+                codePath: scripts/wan2.1_fun/train_track4d360.py
+                codePathLocal: scripts/wan2.1_fun/train_track4d360.py
+                cpu_count: 288
+                cpu_count_logical: 288
+                cudaVersion: "12.7"
+                disk:
+                    /:
+                        total: "255795724288"
+                        used: "546373632"
+                email: yushi.lan@outlook.com
+                executable: /projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12
+                git:
+                    commit: b6681ed2127310d14bc5b93ae5f1d20df8b9e907
+                    remote: git@github.com:NIRVANALAN/track4d-360.git
+                gpu: NVIDIA GH200 120GB
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-fda2ad82-f650-d1df-fbce-bab2bc8dd421
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-f29ba6f4-4da2-cc4b-37dc-d260fa79a222
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-38da6d83-51b2-5586-2248-c681407f2a51
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-aa24fa01-5dca-c359-28f7-7adf38ef534a
+                host: nid010301
+                memory:
+                    total: "919613407232"
+                os: Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38
+                program: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py
+                python: CPython 3.12.13
+                root: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout
+                slurm:
+                    cluster_name: gracehopper
+                    conf: /var/spool/slurmd/conf-cache/slurm.conf
+                    cpu_bind: quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_list: 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_type: 'mask_cpu:'
+                    cpu_bind_verbose: quiet
+                    cpus_on_node: "288"
+                    distribution: cyclic
+                    gpus: "32"
+                    gpus_on_node: "4"
+                    gpus_per_node: "4"
+                    gtids: "0"
+                    job_account: brics.u6ew
+                    job_cpus_per_node: 288(x8)
+                    job_end_time: "1780389885"
+                    job_gid: "1483804847"
+                    job_gpus: 0,1,2,3
+                    job_id: "4982942"
+                    job_name: 14b_pair1x49_profile_ablate_conv3dres0_fsdp2
+                    job_nodelist: nid[010301-010304,010306,010308-010309,010321]
+                    job_num_nodes: "8"
+                    job_partition: workq
+                    job_qos: normal
+                    job_start_time: "1780382685"
+                    job_uid: "1483804847"
+                    job_user: yushi.u6ew
+                    jobid: "4982942"
+                    launch_node_ipaddr: 10.242.5.140
+                    localid: "0"
+                    mpi_type: pmi2
+                    nnodes: "8"
+                    nodeid: "0"
+                    nodelist: nid[010301-010304,010306,010308-010309,010321]
+                    nprocs: "8"
+                    ntasks: "8"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    srun_comm_host: 10.242.5.140
+                    srun_comm_port: "45579"
+                    step_gpus: 0,1,2,3
+                    step_id: "0"
+                    step_launcher_port: "45579"
+                    step_nodelist: nid[010301-010304,010306,010308-010309,010321]
+                    step_num_nodes: "8"
+                    step_num_tasks: "8"
+                    step_resv_ports: 26618-26619
+                    step_tasks_per_node: 1(x8)
+                    stepid: "0"
+                    submit_dir: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360
+                    submit_host: login40
+                    task_pid: "152888"
+                    tasks_per_node: 1(x8)
+                    topology_addr: global.group4.nid010301
+                    topology_addr_pattern: switch.switch.node
+                    umask: "0022"
+                startedAt: "2026-06-02T06:50:12.043060Z"
+                writerId: 1m9b4etgqyfvjqvydz51tku5m7qgycq3
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 98
+                - 106
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+                - 106
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 41
+                - 61
+            "4": 3.12.13
+            "5": 0.26.0
+            "6": 4.46.2
+            "12": 0.26.0
+            "13": linux-aarch64
+allow_existing_checkpoints:
+    value: true
+camera_condition_cfg_dropout_prob:
+    value: 0
+camera_injection_mode:
+    value: projector
+camera_plucker_encoder_mode:
+    value: patchify
+channel_concat_stage:
+    value: null
+cond_drop_dense_prob:
+    value: 0
+cond_drop_geo_prob:
+    value: 0.1
+cond_drop_track_prob:
+    value: 0.1
+data_file_keys:
+    value: image,video
+dataloader_prefetch_factor:
+    value: 2
+dataset_base_path:
+    value: null
+dataset_metadata_path:
+    value: null
+dataset_num_workers:
+    value: 7
+dataset_repeat:
+    value: 1
+debug_latents:
+    value: false
+debug_latents_every:
+    value: 500
+debug_latents_fps:
+    value: 16
+debug_latents_max_frames:
+    value: null
+debug_latents_outdir:
+    value: debug_latents
+debug_latents_quality:
+    value: 5
+debug_latents_vae_tile_size:
+    value:
+        - 30
+        - 52
+debug_latents_vae_tile_stride:
+    value:
+        - 15
+        - 26
+debug_latents_vae_tiled:
+    value: true
+debug_missing_grads:
+    value: false
+dense_alpha_track:
+    value: 1
+dense_conditioning_noise_max_level:
+    value: 0.15
+dense_hybrid_disable_dynamic_masks:
+    value: false
+dense_in_channels:
+    value: 5
+dense_num_residual_blocks:
+    value: 2
+dense_only_condition_modules:
+    value: false
+dense_proj_dim:
+    value: 32
+dense_warp_mode:
+    value: hybrid_softsplat
+dense_warp_motion_threshold:
+    value: 0.005
+disable_track_modules:
+    value: false
+dit_finetune_modules:
+    value: self_attn
+drop_i2v_prob:
+    value: null
+drop_t2v_prob:
+    value: null
+drop_text_prob:
+    value: 0.1
+drop_video_prob:
+    value: 0
+dummy:
+    value: false
+enable_v2v_plucker_camera_control:
+    value: true
+explicit_motion_loss_decode_checkpoint:
+    value: false
+explicit_motion_loss_decode_chunk_size:
+    value: 0
+explicit_motion_loss_snr_sigmoid:
+    value: true
+explicit_motion_loss_snr_sigmoid_a:
+    value: 1
+explicit_motion_loss_snr_sigmoid_b:
+    value: -5
+explicit_motion_loss_snr_weight:
+    value: true
+explicit_motion_loss_weight:
+    value: 0
+extra_inputs:
+    value: null
+find_unused_parameters:
+    value: false
+fixed_num_inference_steps:
+    value: 50
+fixed_sigma:
+    value: null
+fixed_step_frac:
+    value: null
+fp8_models:
+    value: null
+freeze_lora_after_load:
+    value: false
+freeze_track_block_injector:
+    value: false
+freeze_track4d_condition_modules:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gs_abs_depth_loss_weight:
+    value: 0
+gs_assoc_track_2d_loss_weight:
+    value: 0
+gs_depth_loss_weight:
+    value: 0.05
+gs_direct_pos_loss_weight:
+    value: 0
+gs_disable_delta_opacity:
+    value: false
+gs_dyn_depth_loss_weight:
+    value: 0.5
+gs_dynamic_max_t_video:
+    value: 64
+gs_dynamic_mode:
+    value: static
+gs_feat_dim:
+    value: 1024
+gs_head_init_checkpoint:
+    value: null
+gs_head_style:
+    value: flashworld
+gs_legacy_union_render:
+    value: false
+gs_loss_weight:
+    value: 1
+gs_lpips_loss_weight:
+    value: 0.5
+gs_lyra_depth_param:
+    value: exp
+gs_lyra_dfar:
+    value: 10
+gs_lyra_dnear:
+    value: 0.1
+gs_lyra_log_depth_shift:
+    value: 0
+gs_mse_loss_weight:
+    value: 1
+gs_num_points_per_pixel:
+    value: 2
+gs_only_training:
+    value: false
+gs_opa_tv_weight:
+    value: 0.01
+gs_opacity_loss_weight:
+    value: 0.1
+gs_opacity_prune_keep_frac:
+    value: 1
+gs_pos_loss_weight:
+    value: 1
+gs_predict_rotation:
+    value: false
+gs_render_patch_size:
+    value: null
+gs_replay_cache_size:
+    value: 0
+gs_rgb_loss_weight:
+    value: null
+gs_scale_cap:
+    value: null
+gs_sh_degree:
+    value: 2
+gs_track_accel_loss_weight:
+    value: 0
+gs_track_canonical_detach:
+    value: false
+gs_track_canonical_loss_weight:
+    value: 0
+gs_track_delta_loss_weight:
+    value: 0
+gs_track_final_pos_loss_weight:
+    value: 0
+gs_track_joint_render:
+    value: false
+gs_track_loss_quantile:
+    value: 0.98
+gs_track_supervision_mode:
+    value: pointmap
+gs_xyz_tv_weight:
+    value: 0.01
+gs_zero_features:
+    value: false
+height:
+    value: 288
+initial_step:
+    value: 2530
+initialize_model_on_cpu:
+    value: false
+input_video_noise_random_max:
+    value: 500
+input_video_noise_strength:
+    value: 0
+joint_depth_prediction:
+    value: false
+joint_track_depth_prediction:
+    value: false
+joint_track_prediction:
+    value: false
+learning_rate:
+    value: 2e-05
+lora_alpha:
+    value: null
+lora_base_model:
+    value: null
+lora_checkpoint:
+    value: null
+lora_rank:
+    value: 32
+lora_target_modules:
+    value: q,k,v,o,ffn.0,ffn.2
+max_checkpoints_to_keep:
+    value: 5
+max_pixels:
+    value: 1048576
+max_train_steps:
+    value: 2560
+model_id_with_origin_paths:
+    value: Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+model_paths:
+    value: null
+model_variant:
+    value: vt2v
+multi_frame_motion_threshold:
+    value: 0.01
+multi_frame_warped_appearance:
+    value: false
+no_gs_latent_detach:
+    value: false
+normalize_track_with_xyz_cond_stats:
+    value: false
+num_epochs:
+    value: 200
+num_frames:
+    value: 49
+num_workers:
+    value: 7
+offload_models:
+    value: null
+output_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout
+plucker_use_ray_origin:
+    value: false
+pointmap_stats_path:
+    value: null
+predict_track_residual:
+    value: false
+prefetch_factor:
+    value: 2
+prenormalized_track_video_signed_log:
+    value: false
+preset_lora_model:
+    value: null
+preset_lora_path:
+    value: null
+remove_prefix_in_ckpt:
+    value: pipe.dit.
+residual_linear_normalization:
+    value: false
+residual_log_scale_factor:
+    value: 3
+residual_target_xyz_source:
+    value: clean
+residual_zero_center_normalization:
+    value: false
+resume_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+save_every_n_epochs:
+    value: 2
+save_steps:
+    value: 500
+scene:
+    value: null
+sequence_idx:
+    value: null
+skip_diffusion_loss:
+    value: false
+skip_v2v_required_trainables:
+    value: false
+source_mask_boundary_depth_dropout_prob:
+    value: 0.5
+source_mask_boundary_kernel_size:
+    value: 5
+steps_per_epoch:
+    value: 25000
+stream3r_common_scale_stat:
+    value: mean
+syn4d_root:
+    value: /scratch/shared/beegfs/zeren/Syn4D
+target_depth_log_cap:
+    value: 14
+target_depth_max_disparity:
+    value: 5
+target_depth_norm_type:
+    value: log_depth
+task:
+    value: sft:v2v
+tokenizer_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+torch_distributed_debug:
+    value: null
+torch_profile_active_steps:
+    value: 5
+torch_profile_dir:
+    value: null
+torch_profile_memory:
+    value: false
+torch_profile_record_shapes:
+    value: false
+torch_profile_repeat:
+    value: 1
+torch_profile_wait_steps:
+    value: 3
+torch_profile_warmup_steps:
+    value: 1
+torch_profile_with_stack:
+    value: false
+track_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml
+track_injection_block_mode:
+    value: concat_project
+track_injection_mode:
+    value: per_block
+track_spatial_stride:
+    value: 1
+track_xyz_normalization:
+    value: minmax
+track_xyz_quantile_high:
+    value: 0.98
+track_xyz_quantile_low:
+    value: 0.02
+track4d360_data_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+train_plucker_camera_control:
+    value: false
+train_sigma_shift:
+    value: 5
+trainable_models:
+    value: dit.domain_embedding,dit.control_patch_embedding
+use_dense_branch:
+    value: true
+use_dense_null_token_fallback:
+    value: false
+use_depth_null_token:
+    value: false
+use_gen3c_warp_latent_channels:
+    value: false
+use_gradient_checkpointing:
+    value: true
+use_gradient_checkpointing_offload:
+    value: false
+use_gs_feedback_training:
+    value: false
+use_gs_output:
+    value: false
+use_perlayer_camera_control:
+    value: true
+use_plucker_camera_control:
+    value: true
+use_plucker_raymap_tokens:
+    value: true
+use_query_frame_impulse_condition:
+    value: true
+use_residual_tracks:
+    value: true
+use_track_null_token_fallback:
+    value: false
+use_vista4d_latent_streams:
+    value: false
+use_wandb:
+    value: true
+use_xyz_video_condition:
+    value: false
+v2v_channel_patch_embed_mode:
+    value: add
+v2v_concat_mode:
+    value: width
+vxf_init_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+wandb_entity:
+    value: null
+wandb_log_freq:
+    value: 100
+wandb_log_gradients:
+    value: false
+wandb_log_parameters:
+    value: false
+wandb_name:
+    value: ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp
+wandb_notes:
+    value: null
+wandb_project:
+    value: track4d-360
+wandb_tags:
+    value:
+        - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+wandb_watch_model:
+    value: false
+warped_appearance_fusion:
+    value: concat_proj
+warped_condition_mode:
+    value: "off"
+warped_geom_only_dense:
+    value: true
+weight_decay:
+    value: 0.01
+width:
+    value: 512
+xyz_video_condition_aug_prob:
+    value: 0
+xyz_video_condition_cfg_dropout_prob:
+    value: 0
+xyz_video_condition_dropout_prob:
+    value: 0
+xyz_video_condition_frame_scale_std:
+    value: 0
+xyz_video_condition_mode:
+    value: xyz
+xyz_video_condition_noise_std:
+    value: 0
+xyz_video_condition_scale:
+    value: 1
+xyz_video_condition_z_noise_std:
+    value: 0

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/files/output.log

@@ -0,0 +1,234 @@
+✓ WandB initialized: https://wandb.ai/nirvanalan/track4d-360/runs/ixxjov67
+/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/lib/python3.12/site-packages/accelerate/utils/fsdp_utils.py:742: UserWarning: FSDP upcast of low precision parameters to fp32 (since mixed_precision != 'no') may affect the precision of model checkpoints. This effects 593 parameters: ['pipe.dit.domain_embedding', 'pipe.dit.blocks.0.self_attn.q.weight', 'pipe.dit.blocks.0.self_attn.q.bias', 'pipe.dit.blocks.0.self_attn.k.weight', 'pipe.dit.blocks.0.self_attn.k.bias', 'pipe.dit.blocks.0.self_attn.v.weight', 'pipe.dit.blocks.0.self_attn.v.bias', 'pipe.dit.blocks.0.self_attn.o.weight', 'pipe.dit.blocks.0.self_attn.o.bias', 'pipe.dit.blocks.0.self_attn.norm_q.weight', 'pipe.dit.blocks.0.self_attn.norm_k.weight', 'pipe.dit.blocks.0.camera_projector.weight', 'pipe.dit.blocks.0.camera_projector.bias', 'pipe.dit.blocks.1.self_attn.q.weight', 'pipe.dit.blocks.1.self_attn.q.bias', 'pipe.dit.blocks.1.self_attn.k.weight', 'pipe.dit.blocks.1.self_attn.k.bias', 'pipe.dit.blocks.1.self_attn.v.weight', 'pipe.dit.blocks.1.self_attn.v.bias', 'pipe.dit.blocks.1.self_attn.o.weight', 'pipe.dit.blocks.1.self_attn.o.bias', 'pipe.dit.blocks.1.self_attn.norm_q.weight', 'pipe.dit.blocks.1.self_attn.norm_k.weight', 'pipe.dit.blocks.1.camera_projector.weight', 'pipe.dit.blocks.1.camera_projector.bias', 'pipe.dit.blocks.2.self_attn.q.weight', 'pipe.dit.blocks.2.self_attn.q.bias', 'pipe.dit.blocks.2.self_attn.k.weight', 'pipe.dit.blocks.2.self_attn.k.bias', 'pipe.dit.blocks.2.self_attn.v.weight', 'pipe.dit.blocks.2.self_attn.v.bias', 'pipe.dit.blocks.2.self_attn.o.weight', 'pipe.dit.blocks.2.self_attn.o.bias', 'pipe.dit.blocks.2.self_attn.norm_q.weight', 'pipe.dit.blocks.2.self_attn.norm_k.weight', 'pipe.dit.blocks.2.camera_projector.weight', 'pipe.dit.blocks.2.camera_projector.bias', 'pipe.dit.blocks.3.self_attn.q.weight', 'pipe.dit.blocks.3.self_attn.q.bias', 'pipe.dit.blocks.3.self_attn.k.weight', 'pipe.dit.blocks.3.self_attn.k.bias', 'pipe.dit.blocks.3.self_attn.v.weight', 'pipe.dit.blocks.3.self_attn.v.bias', 'pipe.dit.blocks.3.self_attn.o.weight', 'pipe.dit.blocks.3.self_attn.o.bias', 'pipe.dit.blocks.3.self_attn.norm_q.weight', 'pipe.dit.blocks.3.self_attn.norm_k.weight', 'pipe.dit.blocks.3.camera_projector.weight', 'pipe.dit.blocks.3.camera_projector.bias', 'pipe.dit.blocks.4.self_attn.q.weight', 'pipe.dit.blocks.4.self_attn.q.bias', 'pipe.dit.blocks.4.self_attn.k.weight', 'pipe.dit.blocks.4.self_attn.k.bias', 'pipe.dit.blocks.4.self_attn.v.weight', 'pipe.dit.blocks.4.self_attn.v.bias', 'pipe.dit.blocks.4.self_attn.o.weight', 'pipe.dit.blocks.4.self_attn.o.bias', 'pipe.dit.blocks.4.self_attn.norm_q.weight', 'pipe.dit.blocks.4.self_attn.norm_k.weight', 'pipe.dit.blocks.4.camera_projector.weight', 'pipe.dit.blocks.4.camera_projector.bias', 'pipe.dit.blocks.5.self_attn.q.weight', 'pipe.dit.blocks.5.self_attn.q.bias', 'pipe.dit.blocks.5.self_attn.k.weight', 'pipe.dit.blocks.5.self_attn.k.bias', 'pipe.dit.blocks.5.self_attn.v.weight', 'pipe.dit.blocks.5.self_attn.v.bias', 'pipe.dit.blocks.5.self_attn.o.weight', 'pipe.dit.blocks.5.self_attn.o.bias', 'pipe.dit.blocks.5.self_attn.norm_q.weight', 'pipe.dit.blocks.5.self_attn.norm_k.weight', 'pipe.dit.blocks.5.camera_projector.weight', 'pipe.dit.blocks.5.camera_projector.bias', 'pipe.dit.blocks.6.self_attn.q.weight', 'pipe.dit.blocks.6.self_attn.q.bias', 'pipe.dit.blocks.6.self_attn.k.weight', 'pipe.dit.blocks.6.self_attn.k.bias', 'pipe.dit.blocks.6.self_attn.v.weight', 'pipe.dit.blocks.6.self_attn.v.bias', 'pipe.dit.blocks.6.self_attn.o.weight', 'pipe.dit.blocks.6.self_attn.o.bias', 'pipe.dit.blocks.6.self_attn.norm_q.weight', 'pipe.dit.blocks.6.self_attn.norm_k.weight', 'pipe.dit.blocks.6.camera_projector.weight', 'pipe.dit.blocks.6.camera_projector.bias', 'pipe.dit.blocks.7.self_attn.q.weight', 'pipe.dit.blocks.7.self_attn.q.bias', 'pipe.dit.blocks.7.self_attn.k.weight', 'pipe.dit.blocks.7.self_attn.k.bias', 'pipe.dit.blocks.7.self_attn.v.weight', 'pipe.dit.blocks.7.self_attn.v.bias', 'pipe.dit.blocks.7.self_attn.o.weight', 'pipe.dit.blocks.7.self_attn.o.bias', 'pipe.dit.block
+  warnings.warn(
+
+================================================================================
+TRAINABLE PARAMETERS VERIFICATION
+================================================================================
+
+Trainable modules (94 module groups):
+
+  📦 pipe.dit.blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+     pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.dense_target_control_encoder.residual_blocks:
+     Total params: 110,720
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 1.conv1.weight: 27,648
+         ├─ 1.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+         ├─ 1.conv1.bias: 32
+         ├─ 1.conv2.bias: 32
+
+  📦 pipe.dit._camera_patchify_conv:
+     Total params: 36,870
+         ├─ weight: 36,864
+         ├─ bias: 6
+
+  📦 pipe.dit._camera_plucker_patchify_encoder:
+     Total params: 35,840
+         ├─ weight: 30,720
+         ├─ bias: 5,120
+
+  📦 pipe.dit.control_patch_embedding:
+     Total params: 332,800
+         ├─ weight: 327,680
+         ├─ bias: 5,120
+
+  📦 pipe.dit.domain_embedding:
+     Total params: 10,240
+     pipe.dit.domain_embedding: 10,240
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+     pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         ├─ 16.weight: 27,013,120
+         ├─ 17.weight: 27,013,120
+         ├─ 18.weight: 27,013,120
+         ├─ 19.weight: 27,013,120
+         ├─ 2.weight: 27,013,120
+         ├─ 20.weight: 27,013,120
+         ├─ 21.weight: 27,013,120
+         ├─ 22.weight: 27,013,120
+         ├─ 23.weight: 27,013,120
+         ├─ 24.weight: 27,013,120
+         ├─ 25.weight: 27,013,120
+         ├─ 26.weight: 27,013,120
+         ├─ 27.weight: 27,013,120
+         ├─ 28.weight: 27,013,120
+         ├─ 29.weight: 27,013,120
+         ├─ 3.weight: 27,013,120
+         ├─ 30.weight: 27,013,120
+         ├─ … (55 more)
+
+────────────────────────────────────────────────────────────────────────────────
+NOTE: FSDP is active; counts below are from the rank-local post-wrap parameter view, not a full global parameter census.
+Total trainable parameters: 6,325,741,170 (6325.74M)
+Total frozen parameters:    10,363,571,200 (10363.57M)
+Percentage trainable (overall): 37.90%
+Percentage trainable (in requested modules: dit): 33.60%
+================================================================================
+
+[torch-profiler] rank0 enabled (wait=3, warmup=1, active=5, repeat=1) -> /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/torch_profiler
+Epoch 1/200:   0%|          | 29/7813 [09:35<42:55:34, 19.85s/it, opt_step=2559, loss=0.0606]  
+[Track4D-360] First train batch runtime flags: use_gradient_checkpointing=True, use_gradient_checkpointing_offload=False
+[track4d-runtime] rank0 patched DiTBlock.forward saw mode=track block_idx=0
+[Attention] Using: Flash Attention 2 (flash_attn)
+[soft_splat_warp] WARN: tracks grid 160x294 (N=47040) != image grid 288x512 (N=147456); bilinear-resampling tracks_world to image grid. This warning fires once per process.
+
+[Training] Reached max_train_steps=2560. Stopping training.
+
+[Checkpoint] Saving 593 parameters (total 6,325,741,170 ≈ 6325.74M):
+
+  📦 blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 _camera_patchify_conv.bias:
+     Total params: 6
+         _camera_patchify_conv.bias: 6
+
+  📦 _camera_patchify_conv.weight:
+     Total params: 36,864
+         _camera_patchify_conv.weight: 36,864
+
+  📦 _camera_plucker_patchify_encoder.bias:
+     Total params: 5,120
+         _camera_plucker_patchify_encoder.bias: 5,120
+
+  📦 _camera_plucker_patchify_encoder.weight:
+     Total params: 30,720
+         _camera_plucker_patchify_encoder.weight: 30,720
+
+  📦 control_patch_embedding.bias:
+     Total params: 5,120
+         control_patch_embedding.bias: 5,120
+
+  📦 control_patch_embedding.weight:
+     Total params: 327,680
+         control_patch_embedding.weight: 327,680
+
+  📦 domain_embedding:
+     Total params: 10,240
+         domain_embedding: 10,240
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+         pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.dense_target_control_encoder.residual_blocks:
+     Total params: 110,720
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 1.conv1.weight: 27,648
+         ├─ 1.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+         ├─ 1.conv1.bias: 32
+         ├─ 1.conv2.bias: 32
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+         pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         … (72 more)

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/files/requirements.txt

@@ -0,0 +1,208 @@
+videox-fun==1.0.1
+track4d-360==0.1.0
+func_timeout==4.3.5
+click==8.3.2
+pip==26.0.1
+nvidia-cuda-nvrtc-cu12==12.6.77
+ImageIO==2.37.3
+numpy==1.26.4
+boto3==1.42.91
+albucore==0.0.24
+lpips==0.1.4
+torchsde==0.2.6
+tomesd==0.1.3
+yarl==1.23.0
+typer==0.24.1
+triton==3.5.0
+cycler==0.12.1
+multidict==6.7.1
+OpenEXR==3.4.10
+soxr==1.0.0
+kiwisolver==1.5.0
+simsimd==6.5.16
+parso==0.8.6
+einops==0.8.2
+ftfy==6.3.1
+platformdirs==4.9.6
+groovy==0.1.2
+nvidia-nvtx-cu12==12.6.77
+sentry-sdk==2.58.0
+certifi==2026.2.25
+markdown-it-py==4.0.0
+uvicorn==0.38.0
+nvidia-cudnn-cu12==9.10.2.21
+natsort==8.4.0
+awscrt==0.31.2
+accelerate==1.13.0
+deepspeed==0.18.9
+matplotlib==3.10.8
+mdurl==0.1.2
+regex==2026.4.4
+nvidia-cuda-cupti-cu12==12.6.80
+shellingham==1.5.4
+joblib==1.5.3
+zipp==3.23.1
+ipython_pygments_lexers==1.1.1
+executing==2.2.1
+safetensors==0.7.0
+typing-inspection==0.4.2
+httpcore==1.0.9
+semantic-version==2.10.0
+wandb==0.26.0
+asttokens==3.0.1
+datasets==4.8.4
+decorator==5.2.1
+attrs==26.1.0
+six==1.17.0
+importlib_metadata==9.0.0
+py-cpuinfo==9.0.0
+requests==2.33.1
+xxhash==3.6.0
+pytorch-lightning==2.6.1
+fsspec==2026.2.0
+albumentations==2.0.8
+soundfile==0.13.1
+torchaudio==2.9.0
+torchmetrics==1.9.0
+nvidia-cusolver-cu12==11.7.1.2
+scipy==1.17.1
+jmespath==1.1.0
+threadpoolctl==3.6.0
+plyfile==1.1.2
+peft==0.19.1
+pycparser==3.0
+Markdown==3.10.2
+aiosignal==1.4.0
+pydantic==2.13.2
+decord2==3.3.0
+safehttpx==0.1.7
+scikit-image==0.26.0
+fonttools==4.62.1
+lazy-loader==0.5
+smmap==5.0.3
+starlette==1.0.0
+Werkzeug==3.1.8
+propcache==0.4.1
+tensorboard==2.20.0
+rich==15.0.0
+modelscope==1.36.3
+absl-py==2.4.0
+traitlets==5.14.3
+networkx==3.6.1
+hf-xet==1.4.3
+h5py==3.16.0
+PyYAML==6.0.3
+Jinja2==3.1.6
+nvidia-cusparse-cu12==12.5.4.2
+idna==3.11
+transformers==4.46.2
+httpx==0.28.1
+MarkupSafe==3.0.3
+tensorboard-data-server==0.7.2
+pooch==1.9.0
+torch==2.9.0+cu126
+soupsieve==2.8.3
+numba==0.65.0
+torchdiffeq==0.2.5
+jedi==0.19.2
+charset-normalizer==3.4.7
+annotated-doc==0.0.4
+gradio==6.12.0
+Pygments==2.20.0
+pandas==3.0.2
+stack-data==0.6.3
+contourpy==1.3.3
+tomlkit==0.14.0
+scikit-learn==1.8.0
+librosa==0.11.0
+nvidia-cusparselt-cu12==0.7.1
+pytz==2026.1.post1
+go-with-the-track==0.1.0
+yunchang==0.6.4
+onnxruntime==1.24.4
+ninja==1.13.0
+sentencepiece==0.2.1
+pexpect==4.9.0
+multiprocess==0.70.19
+omegaconf==2.3.0
+h11==0.16.0
+pydub==0.25.1
+nvidia-cufft-cu12==11.3.0.4
+xfuser==0.4.5
+tifffile==2026.3.3
+msgpack==1.1.2
+psutil==7.2.2
+ipython==9.12.0
+hf-gradio==0.4.1
+diffusers==0.33.1
+python-dateutil==2.9.0.post0
+orjson==3.11.8
+huggingface_hub==0.36.2
+nvidia-cuda-runtime-cu12==12.6.77
+beautifulsoup4==4.14.3
+ptyprocess==0.7.0
+protobuf==7.34.1
+GitPython==3.1.46
+sympy==1.14.0
+packaging==26.1
+aiohttp==3.13.5
+nvidia-nvshmem-cu12==3.3.20
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+gradio_client==2.4.1
+stringzilla==4.6.0
+flatbuffers==25.12.19
+filelock==3.28.0
+imageio-ffmpeg==0.6.0
+fastapi==0.136.0
+nvidia-nvjitlink-cu12==12.6.85
+s3transfer==0.16.0
+python-multipart==0.0.26
+DistVAE==0.0.0b5
+urllib3==2.6.3
+nvidia-nccl-cu12==2.27.5
+av==15.1.0
+pure_eval==0.2.3
+wheel==0.46.3
+dill==0.4.1
+flash_attn==2.8.3
+track4d-360==0.1.0
+opencv-python==4.11.0.86
+anyio==4.13.0
+pytest==9.0.3
+nvidia-cufile-cu12==1.11.1.6
+lightning-utilities==0.15.3
+pluggy==1.6.0
+grpcio==1.80.0
+cffi==2.0.0
+iniconfig==2.3.0
+videox-fun==1.0.1
+matplotlib-inline==0.2.1
+tokenizers==0.20.3
+pydantic_core==2.46.2
+aiohappyeyeballs==2.6.1
+typing_extensions==4.15.0
+timm==1.0.26
+pyarrow==23.0.1
+mpmath==1.3.0
+prompt_toolkit==3.0.52
+mediapy==1.2.6
+llvmlite==0.47.0
+opencv-python-headless==4.11.0.86
+hjson==3.1.0
+tqdm==4.67.3
+torchvision==0.24.0
+pyparsing==3.3.2
+nvidia-cublas-cu12==12.6.4.1
+botocore==1.42.91
+nanobind==2.9.2
+wcwidth==0.6.0
+gitdb==4.0.12
+frozenlist==1.8.0
+brotli==1.2.0
+trampoline==0.1.2
+pillow==12.2.0
+audioread==3.1.0
+lightning==2.6.1
+setuptools==82.0.1
+nvidia-curand-cu12==10.3.7.77

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/files/wandb-metadata.json

@@ -0,0 +1,235 @@
+{
+  "os": "Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-06-02T06:50:12.043060Z",
+  "args": [
+    "--model_id_with_origin_paths",
+    "Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth",
+    "--tokenizer_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl",
+    "--task",
+    "sft:v2v",
+    "--model_variant",
+    "vt2v",
+    "--v2v_concat_mode",
+    "width",
+    "--height",
+    "288",
+    "--width",
+    "512",
+    "--num_frames",
+    "49",
+    "--track4d360_data_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml",
+    "--steps_per_epoch",
+    "25000",
+    "--num_epochs",
+    "200",
+    "--learning_rate",
+    "2e-5",
+    "--gradient_accumulation_steps",
+    "1",
+    "--save_steps",
+    "500",
+    "--remove_prefix_in_ckpt",
+    "pipe.dit.",
+    "--trainable_models",
+    "dit.domain_embedding,dit.control_patch_embedding",
+    "--dit_finetune_modules",
+    "self_attn",
+    "--track_injection_mode",
+    "per_block",
+    "--track_injection_block_mode",
+    "concat_project",
+    "--track_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml",
+    "--train_sigma_shift",
+    "5.0",
+    "--use_query_frame_impulse_condition",
+    "--use_plucker_camera_control",
+    "--enable_v2v_plucker_camera_control",
+    "--use_plucker_raymap_tokens",
+    "--use_perlayer_camera_control",
+    "--camera_injection_mode",
+    "projector",
+    "--camera_plucker_encoder_mode",
+    "patchify",
+    "--use_dense_branch",
+    "--dense_in_channels",
+    "5",
+    "--dense_proj_dim",
+    "32",
+    "--dense_num_residual_blocks",
+    "2",
+    "--dense_alpha_track",
+    "1.0",
+    "--dense_conditioning_noise_max_level",
+    "0.15",
+    "--source_mask_boundary_depth_dropout_prob",
+    "0.5",
+    "--source_mask_boundary_kernel_size",
+    "5",
+    "--dense_warp_mode",
+    "hybrid_softsplat",
+    "--dense_warp_motion_threshold",
+    "0.005",
+    "--cond_drop_geo_prob",
+    "0.1",
+    "--cond_drop_track_prob",
+    "0.1",
+    "--cond_drop_dense_prob",
+    "0.0",
+    "--initial_step",
+    "2530",
+    "--output_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout",
+    "--dataset_num_workers",
+    "7",
+    "--dataloader_prefetch_factor",
+    "2",
+    "--drop_text_prob",
+    "0.1",
+    "--debug_latents_every",
+    "500",
+    "--allow_existing_checkpoints",
+    "--wandb_project",
+    "track4d-360",
+    "--wandb_name",
+    "ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp",
+    "--wandb_tags",
+    "ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug",
+    "--use_wandb",
+    "--vxf_init_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors",
+    "--use_gradient_checkpointing",
+    "--resume_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors",
+    "--max_train_steps",
+    "2560",
+    "--torch_profile_active_steps",
+    "5",
+    "--torch_profile_wait_steps",
+    "3",
+    "--torch_profile_warmup_steps",
+    "1",
+    "--torch_profile_repeat",
+    "1",
+    "--no_torch_profile_record_shapes",
+    "--no_torch_profile_memory"
+  ],
+  "program": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py",
+  "codePath": "scripts/wan2.1_fun/train_track4d360.py",
+  "codePathLocal": "scripts/wan2.1_fun/train_track4d360.py",
+  "git": {
+    "remote": "git@github.com:NIRVANALAN/track4d-360.git",
+    "commit": "b6681ed2127310d14bc5b93ae5f1d20df8b9e907"
+  },
+  "email": "yushi.lan@outlook.com",
+  "root": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout",
+  "host": "nid010301",
+  "executable": "/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12",
+  "cpu_count": 288,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA GH200 120GB",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "255795724288",
+      "used": "546373632"
+    }
+  },
+  "memory": {
+    "total": "919613407232"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-fda2ad82-f650-d1df-fbce-bab2bc8dd421"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-f29ba6f4-4da2-cc4b-37dc-d260fa79a222"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-38da6d83-51b2-5586-2248-c681407f2a51"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-aa24fa01-5dca-c359-28f7-7adf38ef534a"
+    }
+  ],
+  "cudaVersion": "12.7",
+  "slurm": {
+    "cluster_name": "gracehopper",
+    "conf": "/var/spool/slurmd/conf-cache/slurm.conf",
+    "cpu_bind": "quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_list": "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_type": "mask_cpu:",
+    "cpu_bind_verbose": "quiet",
+    "cpus_on_node": "288",
+    "distribution": "cyclic",
+    "gpus": "32",
+    "gpus_on_node": "4",
+    "gpus_per_node": "4",
+    "gtids": "0",
+    "job_account": "brics.u6ew",
+    "job_cpus_per_node": "288(x8)",
+    "job_end_time": "1780389885",
+    "job_gid": "1483804847",
+    "job_gpus": "0,1,2,3",
+    "job_id": "4982942",
+    "job_name": "14b_pair1x49_profile_ablate_conv3dres0_fsdp2",
+    "job_nodelist": "nid[010301-010304,010306,010308-010309,010321]",
+    "job_num_nodes": "8",
+    "job_partition": "workq",
+    "job_qos": "normal",
+    "job_start_time": "1780382685",
+    "job_uid": "1483804847",
+    "job_user": "yushi.u6ew",
+    "jobid": "4982942",
+    "launch_node_ipaddr": "10.242.5.140",
+    "localid": "0",
+    "mpi_type": "pmi2",
+    "nnodes": "8",
+    "nodeid": "0",
+    "nodelist": "nid[010301-010304,010306,010308-010309,010321]",
+    "nprocs": "8",
+    "ntasks": "8",
+    "ntasks_per_node": "1",
+    "oom_kill_step": "0",
+    "prio_process": "0",
+    "procid": "0",
+    "srun_comm_host": "10.242.5.140",
+    "srun_comm_port": "45579",
+    "step_gpus": "0,1,2,3",
+    "step_id": "0",
+    "step_launcher_port": "45579",
+    "step_nodelist": "nid[010301-010304,010306,010308-010309,010321]",
+    "step_num_nodes": "8",
+    "step_num_tasks": "8",
+    "step_resv_ports": "26618-26619",
+    "step_tasks_per_node": "1(x8)",
+    "stepid": "0",
+    "submit_dir": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360",
+    "submit_host": "login40",
+    "task_pid": "152888",
+    "tasks_per_node": "1(x8)",
+    "topology_addr": "global.group4.nid010301",
+    "topology_addr_pattern": "switch.switch.node",
+    "umask": "0022"
+  },
+  "writerId": "1m9b4etgqyfvjqvydz51tku5m7qgycq3"
+}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/track/injector/block_19/weight_norm":71.72638702392578,"train/track/injector/block_12/grad_norm":0.005037492141127586,"train/timing/unit/WanVideoUnit_Vista4DStreams_s":4.249280391377397e-05,"train/track/injector/block_16/weight_norm":71.75887298583984,"train/track/injector/block_14/weight_norm":71.7538833618164,"train/track/injector/block_06/weight_norm":71.78463745117188,"train/timing/unit/WanVideoUnit_TrackPointCondition_s":0.02038385346531868,"train/track/injector/block_03/grad_norm":0.008266760036349297,"train/track/injector/grad_norm_max":0.025773145258426666,"train/track/injector/block_38/weight_norm":71.60935974121094,"train/timing/data_wait_s":0.0011231391690671444,"train/track/injector/block_07/grad_norm":0.006979101803153753,"train/timing/unit/WanVideoUnit_CameraControl_NVS_s":0.003995758481323719,"train/track/injector/block_09/grad_norm":0.006004222668707371,"train/timing/unit/WanVideoUnit_ShapeChecker_s":4.563275069813244e-05,"train/timing/dit_loss_fn_s":2.957465648651123,"train/loss":0.060648560523986816,"train/track/injector/block_27/weight_norm":71.7016830444336,"train/track/injector/block_31/weight_norm":71.70208740234375,"train/hbm/reserved_gb":39.54953384399414,"train/track/injector/block_32/weight_norm":71.66666412353516,"train/track/injector/block_04/grad_norm":0.007650282233953476,"train/track/injector/block_23/weight_norm":71.71662902832031,"train/track/injector/block_25/weight_norm":71.7304916381836,"train/track/injector/block_30/grad_norm":0.002275619190186262,"train/timing/unit/WanVideoUnit_PromptEmbedder_s":0.00026011947193183005,"train/track/injector/block_22/weight_norm":71.71588897705078,"train/track/injector/block_00/grad_norm":0.025773145258426666,"train/track/injector/block_16/grad_norm":0.005171981640160084,"train/track/injector/block_05/weight_norm":71.7911376953125,"train/track/injector/block_13/weight_norm":71.7365493774414,"train/track/injector/block_37/grad_norm":0.0021059231366962194,"train/track/injector/block_05/grad_norm":0.007884364575147629,"train/track/injector/block_08/weight_norm":71.7770767211914,"train/track/injector/block_15/weight_norm":71.72616577148438,"train/timing/reduction_s":0.10025622975081205,"train/track/injector/block_00/weight_norm":71.61991882324219,"train/cond_dropout/drop_dense":0.09375,"train/track/injector/block_22/grad_norm":0.004614893347024918,"train/track/injector/block_02/grad_norm":0.008216189220547676,"train/timing/step_total_s":13.46668770769611,"train/track/injector/block_13/grad_norm":0.005134806968271732,"train/track/injector/block_12/weight_norm":71.75670623779297,"train/track/injector/block_07/weight_norm":71.78170013427734,"train/track/injector/block_14/grad_norm":0.005121688358485699,"train/track/injector/block_06/grad_norm":0.007252744399011135,"_wandb":{"runtime":598},"train/track/injector/block_37/weight_norm":71.64714813232422,"train/timing/unit/WanVideoUnit_TargetVideoEncoder_s":0.8007816672325134,"train/track/injector/block_01/grad_norm":0.008102822117507458,"train/track/injector/block_21/grad_norm":0.003961995244026184,"train/track/injector/block_30/weight_norm":71.7083511352539,"train/track/injector/block_20/grad_norm":0.0034632328897714615,"train/track/injector/block_15/grad_norm":0.004622212611138821,"train/track/injector/weight_norm_mean":71.7210922241211,"train/track/injector/block_39/grad_norm":0.005907048471271992,"train/track/injector/block_36/weight_norm":71.649658203125,"train/hbm/allocated_gb":3.435471534729004,"train/track/adapter/grad_norm":0.004346003755927086,"train/track/injector/block_32/grad_norm":0.0021721147932112217,"train/track/injector/block_23/grad_norm":0.0035610890481621027,"train/track/injector/block_28/weight_norm":71.74327850341797,"train/timing/unit/WanVideoUnit_TeaCache_s":1.821445403038524e-05,"train/track/injector/block_04/weight_norm":71.78107452392578,"train/track/injector/block_09/weight_norm":71.78056335449219,"train/hbm/post_t5_allocated_gb":28.869243621826172,"train/track/injector/block_31/grad_norm":0.0020449731964617968,"train/track/injector/block_38/grad_norm":0.002583117689937353,"train/timing/unit/WanVideoUnit_DenseTargetControl_s":0.2740522027015686,"train/track/injector/block_03/weight_norm":71.80154418945312,"train/timing/unit/WanVideoUnit_V2VConcatenate_s":0.00011662059841910377,"train/track/injector/block_33/grad_norm":0.002395837800577283,"train/track/injector/block_24/weight_norm":71.72344207763672,"train/t5/on_cuda":-1,"train/timing/unit/WanVideoUnit_LatentNoiseAugmentation_s":4.298090061638504e-05,"train/track/injector/weight_norm_max":71.80154418945312,"train/track/injector/block_35/weight_norm":71.68324279785156,"train/track/injector/block_21/weight_norm":71.71271514892578,"train/cond_dropout/drop_tracks":0.15625,"_runtime":598,"train/timing/optimizer_s":0.031879177782684565,"train/track/injector/grad_norm_mean":0.005017243325710297,"train/track/injector/block_18/grad_norm":0.004682852420955896,"train/track/injector/block_11/grad_norm":0.005184764973819256,"train/timing/unit/WanVideoUnit_WarpedAppearance_s":4.666278255172074e-05,"train/track/injector/block_36/grad_norm":0.0020740143954753876,"train/epoch":0,"train/timing/unit/WanVideoUnit_NoiseInitializer_s":0.00017827811825554818,"train/track/injector/block_35/grad_norm":0.0019575022161006927,"train/track/injector/block_34/grad_norm":0.00215382338501513,"train/timing/unit/WanVideoUnit_InputVideoEmbedder_XYZCondition_s":1.6093270778656006,"train/timing/unit/WanVideoUnit_CfgMerger_s":1.1475616702227853e-05,"train/timing/microsteps":1,"train/track/injector/block_25/grad_norm":0.0028981242794543505,"train/track/injector/block_29/weight_norm":71.7083969116211,"train/track/injector/block_34/weight_norm":71.64429473876953,"train/track/injector/block_26/weight_norm":71.73137664794922,"_step":2559,"train/track/injector/block_39/weight_norm":71.61332702636719,"train/track/injector/block_17/grad_norm":0.004457436036318541,"train/track/injector/block_33/weight_norm":71.65998077392578,"train/track/injector/block_17/weight_norm":71.72579956054688,"train/track/injector/block_26/grad_norm":0.002922400366514921,"train/track/injector/block_10/grad_norm":0.0058774868957698345,"train/track/injector/block_20/weight_norm":71.71338653564453,"train/hbm/post_t5_reserved_gb":28.94489097595215,"train/track/injector/block_19/grad_norm":0.00396856339648366,"train/track/injector/block_11/weight_norm":71.75855255126953,"train/timing/other_s":1.90594895882532,"_timestamp":1.7803835989144747e+09,"train/track/injector/block_27/grad_norm":0.0028194861952215433,"train/track/injector/block_24/grad_norm":0.0033174175769090652,"train/timing/unit/WanVideoUnit_UnifiedSequenceParallel_s":2.4413131541223265e-05,"train/track/injector/block_29/grad_norm":0.0028786961920559406,"train/track/injector/block_18/weight_norm":71.71785736083984,"train/timing/unit/WanVideoUnit_ModeDropoutAugmentation_s":2.5872304831864312e-05,"train/timing/backward_s":5.718927487265319,"train/track/injector/block_08/grad_norm":0.006412863731384277,"train/track/adapter/weight_norm":16.3490047454834,"train/timing/forward_s":5.709675854071975,"train/track/injector/block_01/weight_norm":71.73580932617188,"train/track/injector/block_02/weight_norm":71.77042388916016,"train/track/injector/block_10/weight_norm":71.77764129638672,"train/track/injector/block_28/grad_norm":0.0027806407306343317}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/logs/debug-internal.log

@@ -0,0 +1,97 @@
+{"time":"2026-06-02T06:50:13.570807992Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T06:50:13.573515504Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T06:50:14.064306628Z","level":"INFO","msg":"stream: created new stream","id":"ixxjov67"}
+{"time":"2026-06-02T06:50:14.064412769Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T06:50:14.065276714Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T06:50:14.065282794Z","level":"INFO","msg":"writer: started","stream_id":"ixxjov67"}
+{"time":"2026-06-02T06:50:14.06566832Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T06:50:15.631914228Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T06:50:16.106065701Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:23.169926895Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T06:50:23.18417797Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":175}
+{"time":"2026-06-02T06:50:30.632687583Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":126,"uploaded_len":2}
+{"time":"2026-06-02T06:50:31.057643889Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:50:45.633956602Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T06:50:45.995092564Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:00.632732136Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-06-02T06:51:01.013164986Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:15.632693827Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
+{"time":"2026-06-02T06:51:15.973083783Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:30.632614271Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
+{"time":"2026-06-02T06:51:30.902457359Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:45.632638154Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":127,"console_lines":3}
+{"time":"2026-06-02T06:51:46.012081416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:00.632782948Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:01.245405887Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:15.633245252Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:16.031118944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:30.632460243Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":16,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:31.064836975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:45.632586315Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:52:45.920833256Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:00.632560879Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:01.114903023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:15.633256291Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":22,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:16.048155401Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:30.633585048Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:31.116840996Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:45.632811032Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:53:45.995726219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:00.632163343Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":130,"console_lines":1}
+{"time":"2026-06-02T06:54:01.04412747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:15.632919183Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T06:54:16.033687178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:30.632581426Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T06:54:31.115164341Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:45.631988187Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
+{"time":"2026-06-02T06:54:46.16067707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:00.632478441Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
+{"time":"2026-06-02T06:55:01.137467668Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:15.634021135Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
+{"time":"2026-06-02T06:55:16.528050934Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:30.633071312Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:55:30.990818571Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:45.632948623Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:55:45.947339192Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:00.63331661Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:01.090740183Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:15.633357213Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:16.100888881Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:30.633508896Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:31.248935786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:45.633656982Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:56:46.023339526Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:00.633242778Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:01.000390992Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:15.632535397Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":2,"events_offset":54,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:16.015127404Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:30.633130173Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:30.99356342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:45.633329584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:57:46.07541001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:00.632386559Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:01.18224948Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:15.632888861Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":62,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:16.030413718Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:30.633023024Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:31.068636213Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:45.633331848Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:58:46.039435816Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:00.632360803Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":68,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:59:01.039092307Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:15.632705468Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
+{"time":"2026-06-02T06:59:16.085584263Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:30.632579618Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
+{"time":"2026-06-02T06:59:31.365655224Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:45.633048671Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":2,"events_offset":74,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:59:46.003907075Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T07:00:00.633135873Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":76,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T07:00:01.363370609Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T07:00:13.67931977Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T07:00:13.679909675Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":131,"console_lines":103,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T07:00:14.181094862Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T07:00:14.18215301Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T07:00:14.188249365Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T07:00:14.188298388Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T07:00:14.188708233Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T07:00:14.188737768Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/logs/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 06:50:12,052 INFO    MainThread:152969 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 06:50:12,053 INFO    MainThread:152969 [wandb_setup.py:_flush():81] Configure stats pid to 152969
+2026-06-02 06:50:12,053 INFO    MainThread:152969 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 06:50:12,053 INFO    MainThread:152969 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/logs/debug.log
+2026-06-02 06:50:12,054 INFO    MainThread:152969 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/logs/debug-internal.log
+2026-06-02 06:50:12,054 INFO    MainThread:152969 [wandb_init.py:init():848] calling init triggers
+2026-06-02 06:50:12,060 INFO    MainThread:152969 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 2, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth_noconv3dres.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 06:50:12,060 INFO    MainThread:152969 [wandb_init.py:init():896] starting backend
+2026-06-02 06:50:12,777 INFO    MainThread:152969 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 06:50:13,568 INFO    MainThread:152969 [wandb_init.py:init():919] backend started and connected
+2026-06-02 06:50:13,571 INFO    MainThread:152969 [wandb_init.py:init():989] updated telemetry
+2026-06-02 06:50:13,661 INFO    MainThread:152969 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 06:50:14,653 INFO    MainThread:152969 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 06:50:15,600 INFO    MainThread:152969 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 06:50:15,631 INFO    MainThread:152969 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 07:00:12,998 INFO    MainThread:152969 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/ixxjov67
+2026-06-02 07:00:13,001 INFO    MainThread:152969 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 07:00:13,002 INFO    MainThread:152969 [wandb_run.py:_restore():2490] restore
+2026-06-02 07:00:13,002 INFO    MainThread:152969 [wandb_run.py:_restore():2496] restore done
+2026-06-02 07:00:14,187 INFO    MainThread:152969 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_conv3dres0_cond_dropout/wandb/run-20260602_065012-ixxjov67/run-ixxjov67.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:555b9456a92d6560d1e50f8fe5a5f5f82fc801c9f8646cb469ca07ad7b0d41c8
+size 694119

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+#SBATCH --job-name=14b_8node_288x512_pair1x49_next_eb32_lr2e5
+#SBATCH --nodes=8
+#SBATCH --gpus=32
+#SBATCH --gpus-per-node=4
+#SBATCH --ntasks-per-node=1
+#SBATCH --time=24:00:00
+#SBATCH --output=out/exp_%j_%x.out
+
+# Track4D-360 14B FSDP 288x512 next-stage finetune.
+#
+# Main changes from the 288p backendmix OOM-debug launcher:
+#   - new DATA_TAG / output directory;
+#   - warm-start from the latest existing 288p backendmix checkpoint;
+#   - remove most static datasets from DATA_CONFIG;
+#   - add ReCamMaster CoWTracker + square512 source49 + compact pair1x49
+#     VGGT-Omega labels, q1-only by dataset construction.
+#
+# Submit from repo root on Isambard:
+#   sbatch bash_scripts/track4d_360/ismb/14b/sbatch/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh
+
+export NNODES=8
+export MODEL_SIZE=14b
+export CONDA_ENV="${CONDA_ENV:-th29py312}"
+export HEIGHT=288
+export WIDTH=512
+export NUM_FRAMES=49
+export main_process_port="${main_process_port:-29595}"
+
+export WAN_T2V14B_DIR="${WAN_T2V14B_DIR:-${SLURM_SUBMIT_DIR}/weights/Wan2.1-T2V-14B}"
+export VXF_INIT="${VXF_INIT:-${WAN_T2V14B_DIR}/diffusion_pytorch_model*.safetensors}"
+export MODEL_ID_PREFIX="${MODEL_ID_PREFIX:-Wan-AI/Wan2.1-T2V-14B}"
+export TOKENIZER_PATH="${TOKENIZER_PATH:-${WAN_T2V14B_DIR}/google/umt5-xxl}"
+export T5_PATH="${T5_PATH:-${WAN_T2V14B_DIR}/models_t5_umt5-xxl-enc-bf16.pth}"
+export VAE_PATH="${VAE_PATH:-${WAN_T2V14B_DIR}/Wan2.1_VAE.pth}"
+
+# Prefer the current 288p backendmix run as the next-stage warm start. Fall
+# back to the 384p backendmix/raymap-token parents only when no 288p checkpoint exists.
+export BACKENDMIX_288_PARENT_DIR="${BACKENDMIX_288_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout}"
+export BACKENDMIX_384_PARENT_DIR="${BACKENDMIX_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout}"
+export RAYMAP_384_PARENT_DIR="${RAYMAP_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout}"
+# Warm-start from the 288p backendmix run's step-9000 checkpoint — the requested
+# next-stage starting point. (FSDP1/FSDP2 read it identically: warm-start loads
+# the safetensors BEFORE accelerator.prepare, on the unwrapped model, so it is
+# distributed-backend-independent.) Override WARM_START=... to pick another;
+# falls back to latest backendmix / 384p parents if step-9000 is absent.
+if [ -z "${WARM_START:-}" ]; then
+    if [ -f "${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors" ]; then
+        export WARM_START="${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors"
+    else
+        latest="$(find "${BACKENDMIX_288_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        if [ -z "${latest}" ]; then
+            latest="$(find "${BACKENDMIX_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        if [ -z "${latest}" ]; then
+            latest="$(find "${RAYMAP_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        export WARM_START="${latest:-${RAYMAP_384_PARENT_DIR}/step-3500.safetensors}"
+    fi
+fi
+
+if ! compgen -G "${VXF_INIT}" >/dev/null; then
+    echo "ERROR: VXF_INIT not found: ${VXF_INIT}" >&2
+    exit 1
+fi
+[ -d "${TOKENIZER_PATH}" ] || { echo "ERROR: TOKENIZER_PATH not found: ${TOKENIZER_PATH}" >&2; exit 1; }
+[ -f "${T5_PATH}" ] || { echo "ERROR: T5_PATH not found: ${T5_PATH}" >&2; exit 1; }
+[ -f "${VAE_PATH}" ] || { echo "ERROR: VAE_PATH not found: ${VAE_PATH}" >&2; exit 1; }
+
+export TARGET_EFFECTIVE_BATCH=32
+export gradient_accumulation_steps=1
+export learning_rate="${learning_rate:-2e-5}"
+export save_steps="${save_steps:-500}"
+export dataset_num_workers="${dataset_num_workers:-7}"
+export TRACK4D360_STOP_AFTER_NEXT_SAVE="${TRACK4D360_STOP_AFTER_NEXT_SAVE:-0}"
+
+export dit_finetune_modules="${dit_finetune_modules:-self_attn}"
+export DIT_FINETUNE_TAG="${DIT_FINETUNE_TAG:-attn}"
+export use_plucker_raymap_tokens=1
+export train_plucker_camera_control=0
+export PLUCKER_WANDB_TAG="${PLUCKER_WANDB_TAG:-plucker_raymap_tokens}"
+
+export DATA_CONFIG="${DATA_CONFIG:-${SLURM_SUBMIT_DIR}/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml}"
+export DATA_TAG="${DATA_TAG:-mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512}"
+
+export RECAMMASTER_SOURCE49_ROOT="${RECAMMASTER_SOURCE49_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}"
+export RECAMMASTER_PAIR1X49_LABEL_ROOT="${RECAMMASTER_PAIR1X49_LABEL_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}"
+export RECAMMASTER_COW_ROOT="${RECAMMASTER_COW_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}"
+export TRACK4D360_RECAM_TRACK_SAMPLE_MODE="${TRACK4D360_RECAM_TRACK_SAMPLE_MODE:-grid}"
+export TRACK4D360_RECAM_TRACK_STRIDE="${TRACK4D360_RECAM_TRACK_STRIDE:-2}"
+export TRACK4D360_RECAM_TRACK_SAMPLES="${TRACK4D360_RECAM_TRACK_SAMPLES:-0}"
+# Pair1x49 labels currently cover only the priority block: first 750 numeric
+# train scenes inside each K/focal group (e.g. train/f18_aperture10/scene000..).
+export TRACK4D360_RECAM_SCENE_OFFSET_PER_K="${TRACK4D360_RECAM_SCENE_OFFSET_PER_K:-0}"
+export TRACK4D360_RECAM_SCENE_LIMIT_PER_K="${TRACK4D360_RECAM_SCENE_LIMIT_PER_K:-750}"
+
+# Fixed post-compaction track identity count — MUST be set (the 24h/10k-step
+# backendmix run had this at 65536; this launcher previously omitted it). The
+# ReCamMaster CoW tracks are dense (grid stride 2, samples=0 = uncapped), so
+# without a cap N varies per sample and a heavy draw spikes the track adapter's
+# proj_down activation (B,N,T,hidden) -> the observed CUDA OOM at
+# conv3d_track_adapter.py proj_down (1.16 GiB alloc, step ~770). Fixing N also
+# pads small draws to a constant shape, eliminating per-step fragmentation. Lower
+# to 32768 for extra headroom on contended nodes.
+export TRACK4D360_TRACK_IDENTITY_FIXED_N="${TRACK4D360_TRACK_IDENTITY_FIXED_N:-65536}"
+
+export TRACK4D360_SPARSE_TRACK_PROB="${TRACK4D360_SPARSE_TRACK_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_STRIDE="${TRACK4D360_SPARSE_TRACK_STRIDE:-2|4|8}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB="${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_SIZE="${TRACK4D360_SPARSE_TRACK_BLOCK_SIZE:-64}"
+export TRACK4D360_MULTI_QUERY_PROB="${TRACK4D360_MULTI_QUERY_PROB:-1.0}"
+export TRACK4D360_MULTI_QUERY_CHOICES="${TRACK4D360_MULTI_QUERY_CHOICES:-1|2|3|4}"
+export TRACK4D360_GEOMETRY_NOISE_PROB="${TRACK4D360_GEOMETRY_NOISE_PROB:-1.0}"
+export TRACK4D360_TRACK_OFFSET_STD="${TRACK4D360_TRACK_OFFSET_STD:-0.015}"
+export TRACK4D360_TRACK_TEMPORAL_JITTER_STD="${TRACK4D360_TRACK_TEMPORAL_JITTER_STD:-0.006}"
+export TRACK4D360_TRACK_XY_NOISE_RATIO="${TRACK4D360_TRACK_XY_NOISE_RATIO:-0.25}"
+export TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO="${TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO:-0.5}"
+export TRACK4D360_DEPTH_OFFSET_STD="${TRACK4D360_DEPTH_OFFSET_STD:-0.006}"
+export TRACK4D360_DEPTH_SCALE_JITTER="${TRACK4D360_DEPTH_SCALE_JITTER:-0.01}"
+export TRACK4D360_SOURCE_RGB_NOISE_STD="${TRACK4D360_SOURCE_RGB_NOISE_STD:-0.01}"
+export TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD="${TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD:-0.02}"
+export TRACK4D360_DYNAMIC_MASK_AUG_PROB="${TRACK4D360_DYNAMIC_MASK_AUG_PROB:-0.5}"
+export TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL="${TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL:-5}"
+export TRACK4D360_DYNAMIC_MASK_FLIP_PROB="${TRACK4D360_DYNAMIC_MASK_FLIP_PROB:-0.002}"
+
+export dense_warp_mode="${dense_warp_mode:-hybrid_softsplat}"
+export dense_warp_motion_threshold="${dense_warp_motion_threshold:-0.005}"
+export dense_conditioning_noise_max_level="${dense_conditioning_noise_max_level:-0.15}"
+export source_mask_boundary_depth_dropout_prob="${source_mask_boundary_depth_dropout_prob:-0.5}"
+export source_mask_boundary_kernel_size="${source_mask_boundary_kernel_size:-5}"
+export freeze_track4d_condition_modules="${freeze_track4d_condition_modules:-0}"
+
+export use_gradient_checkpointing=1
+export use_gradient_checkpointing_offload="${use_gradient_checkpointing_offload:-0}"
+export VXF_OFFLOAD_T5_TO_CPU="${VXF_OFFLOAD_T5_TO_CPU:-1}"
+export TRACK4D360_MEM_PROBE="${TRACK4D360_MEM_PROBE:-0}"
+export TRACK4D360_ADAPTER_MEM_PROFILE="${TRACK4D360_ADAPTER_MEM_PROFILE:-0}"
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-${PYTORCH_ALLOC_CONF:-expandable_segments:True}}"
+export PYTORCH_ALLOC_CONF="${PYTORCH_ALLOC_CONF:-${PYTORCH_CUDA_ALLOC_CONF}}"
+export TRACK4D360_DENSE_HYBRID_CACHE_FRAMES="${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES:-7}"
+export TRACK4D360_DENSE_HYBRID_TGT_CHUNK="${TRACK4D360_DENSE_HYBRID_TGT_CHUNK:-2}"
+
+export NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-0}"
+export NCCL_P2P_LEVEL="${NCCL_P2P_LEVEL:-NVL}"
+export NCCL_NVLS_ENABLE=0
+export NCCL_NSOCKS_PERTHREAD=8
+export NCCL_SOCKET_NTHREADS=8
+export NCCL_MAX_NCHANNELS="${NCCL_MAX_NCHANNELS:-32}"
+export NCCL_BUFFSIZE="${NCCL_BUFFSIZE:-8388608}"
+export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
+
+export TRACK4D360_CAPTION_OUTPUT_ROOT="${TRACK4D360_CAPTION_OUTPUT_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output}"
+export SYN4D_CAPTION_DIR="${SYN4D_CAPTION_DIR:-${TRACK4D360_CAPTION_OUTPUT_ROOT}/Syn4D}"
+
+echo "[raymap-tokens 14B pair1x49 next] T2V init: ${VXF_INIT}"
+echo "[raymap-tokens 14B pair1x49 next] warm start: ${WARM_START}"
+echo "[raymap-tokens 14B pair1x49 next] data: ${DATA_CONFIG}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam source49: ${RECAMMASTER_SOURCE49_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam pair1x49 labels: ${RECAMMASTER_PAIR1X49_LABEL_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam CoW: ${RECAMMASTER_COW_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam q1 track sampling: mode=${TRACK4D360_RECAM_TRACK_SAMPLE_MODE}, stride=${TRACK4D360_RECAM_TRACK_STRIDE}, samples=${TRACK4D360_RECAM_TRACK_SAMPLES}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam scene window per K: offset=${TRACK4D360_RECAM_SCENE_OFFSET_PER_K}, limit=${TRACK4D360_RECAM_SCENE_LIMIT_PER_K}"
+echo "[raymap-tokens 14B pair1x49 next] q policy: ReCam q1-only; synthetic choices=${TRACK4D360_MULTI_QUERY_CHOICES}"
+echo "[raymap-tokens 14B pair1x49 next] sparse synthetic tracks: stride_choices=${TRACK4D360_SPARSE_TRACK_STRIDE}, block_jitter_prob=${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB}"
+echo "[raymap-tokens 14B pair1x49 next] trainables: self_attn + raymap Ecam/camera_projector + active track/dense condition modules"
+echo "[raymap-tokens 14B pair1x49 next] freeze track/dense condition modules: ${freeze_track4d_condition_modules}"
+echo "[raymap-tokens 14B pair1x49 next] batch: effective=32, grad_accum=1, lr=${learning_rate}, save_steps=${save_steps}"
+echo "[raymap-tokens 14B pair1x49 next] memory: grad_ckpt_offload=${use_gradient_checkpointing_offload}, hybrid_cache_frames=${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES}, hybrid_tgt_chunk=${TRACK4D360_DENSE_HYBRID_TGT_CHUNK}, cuda_alloc_conf=${PYTORCH_CUDA_ALLOC_CONF}"
+
+source "${SLURM_SUBMIT_DIR}/bash_scripts/track4d_360/ismb/sbatch_common.sh"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml

@@ -0,0 +1,77 @@
+# Next-stage 288x512 backendmix finetune with cleaned ReCamMaster pair1x49 labels.
+#
+# Mix: 35% ReCamMaster pair1x49, 35% Syn4D, 5% Kubric49, 5% Kubric81,
+# and 20% legacy static datasets. The static 20% keeps the previous backendmix
+# relative weights: DL3DV:RE10K:MVS-Synth:TartanAir = 65000:7500:12500:25000.
+# ReCamMaster uses compact square512 source49 + pair1x49 VGGT-Omega scale
+# labels plus CoWTracker source tracks and is q1-only by construction.
+#
+# sbatch_common.sh still sanity-checks for the old MVS-Synth guard string:
+# exclude_sequence_ids=['0000']. This config has no MVS-Synth entry.
+
+num_views: 49
+resolution: [512, 288]
+
+syn4d_root: ${oc.env:SYN4D_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/Syn4D}
+syn4d_caption_dir: ${oc.env:SYN4D_CAPTION_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/Syn4D}
+syn4d_valid_sequences: configs/syn4d_valid_sequences.txt
+dl3dv_root: ${oc.env:DL3DV_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/dl3dv_tar}
+re10k_root: ${oc.env:RE10K_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_re10k}
+mvs_synth_root: ${oc.env:MVS_SYNTH_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_mvs_synth}
+tartanair_root: ${oc.env:TARTANAIR_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_tartanair}
+kubric49_root: ${oc.env:KUBRIC49_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_49frames}
+kubric81_root: ${oc.env:KUBRIC81_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_81frames}
+kubric49_caption_cache_dir: ${oc.env:KUBRIC49_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_49}
+kubric81_caption_cache_dir: ${oc.env:KUBRIC81_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_81}
+recammaster_caption_cache_dir: ${oc.env:RECAMMASTER_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/recammaster}
+recammaster_root: ${oc.env:RECAMMASTER_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset}
+multicam_fg_mask_root: ${oc.env:MULTICAM_FG_MASK_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_SAM3_FG_81f_sources_person_human_288x512}
+recammaster_source49_root: ${oc.env:RECAMMASTER_SOURCE49_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}
+recammaster_pair1x49_label_root: ${oc.env:RECAMMASTER_PAIR1X49_LABEL_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}
+recammaster_cow_root: ${oc.env:RECAMMASTER_COW_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}
+recammaster_track_sample_mode: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLE_MODE,grid}
+recammaster_track_stride: ${oc.env:TRACK4D360_RECAM_TRACK_STRIDE,2}
+recammaster_track_samples: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLES,0}
+recammaster_scene_offset_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_OFFSET_PER_K,0}
+recammaster_scene_limit_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_LIMIT_PER_K,750}
+
+sparse_track_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_PROB,1.0}
+sparse_track_stride: ${oc.env:TRACK4D360_SPARSE_TRACK_STRIDE,2|4|8}
+sparse_track_query_stride_mode: ${oc.env:TRACK4D360_SPARSE_TRACK_QUERY_STRIDE_MODE,query_count_floor}
+sparse_track_block_jitter_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB,1.0}
+sparse_track_block_size: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_SIZE,64}
+multi_query_prob: ${oc.env:TRACK4D360_MULTI_QUERY_PROB,1.0}
+multi_query_choices: ${oc.env:TRACK4D360_MULTI_QUERY_CHOICES,1|2|3|4}
+geometry_noise_prob: ${oc.env:TRACK4D360_GEOMETRY_NOISE_PROB,1.0}
+track_offset_std: ${oc.env:TRACK4D360_TRACK_OFFSET_STD,0.015}
+track_temporal_jitter_std: ${oc.env:TRACK4D360_TRACK_TEMPORAL_JITTER_STD,0.006}
+track_xy_noise_ratio: ${oc.env:TRACK4D360_TRACK_XY_NOISE_RATIO,0.25}
+track_frame_depth_jitter_ratio: ${oc.env:TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO,0.5}
+depth_offset_std: ${oc.env:TRACK4D360_DEPTH_OFFSET_STD,0.006}
+depth_scale_jitter: ${oc.env:TRACK4D360_DEPTH_SCALE_JITTER,0.01}
+source_rgb_noise_std: ${oc.env:TRACK4D360_SOURCE_RGB_NOISE_STD,0.01}
+source_rgb_frame_jitter_std: ${oc.env:TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD,0.02}
+dynamic_mask_aug_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_AUG_PROB,0.5}
+dynamic_mask_morph_kernel: ${oc.env:TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL,5}
+dynamic_mask_flip_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_FLIP_PROB,0.002}
+
+train_datasets:
+  # 35% ReCamMaster CoWTracker + square512 source49 + compact pair1x49 VGGT labels.
+  # This dataset is q1 only: it always emits track_query_indices=[0]. The
+  # default track density is grid stride 2 to avoid lifting every CoW pixel
+  # during training; override TRACK4D360_RECAM_TRACK_SAMPLE_MODE=all for dense.
+  - "87500 @ ReCamMasterCowVGGT_NVS(ROOT='${recammaster_root}', vggt_root='${recammaster_source49_root}', source_vggt_root='${recammaster_source49_root}', pair1x49_label_root='${recammaster_pair1x49_label_root}', geometry_source='pair1x49_label_scale_only', cow_root='${recammaster_cow_root}', fg_mask_root='${multicam_fg_mask_root}', caption_cache_dir='${recammaster_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, vggt_post_crop_mode='cow_aspect', rgb_post_crop_mode='cow_aspect', track_sample_mode='${recammaster_track_sample_mode}', track_stride=${recammaster_track_stride}, track_samples=${recammaster_track_samples}, scene_offset_per_k=${recammaster_scene_offset_per_k}, scene_limit_per_k=${recammaster_scene_limit_per_k})"
+  # 35% Syn4D dynamic multi-camera mesh-tracked 3D.
+  - "87500 @ GeometryRobustnessWrapper_NVS(Syn4D_Multi_NVS(ROOT='${syn4d_root}', caption_dir='${syn4d_caption_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, track_spatial_stride=1, exclude_scenes=['flying_group'], valid_sequences_file='${syn4d_valid_sequences}', use_mp4_rgb=True, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 49-frame dynamic data.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric49_root}', caption_cache_dir='${kubric49_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 81-frame dynamic data sampled to 49.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric81_root}', caption_cache_dir='${kubric81_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 11.82% DL3DV (real, DA3 depth). Static total is 20%; this keeps the old static relative weight.
+  - "29550 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${dl3dv_root}', dataset_type='dl3dv', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=29550, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=0.5, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, use_da3_depth=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 1.38% RE10K (real, DA3 depth).
+  - "3450 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${re10k_root}', dataset_type='re10k', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=3450, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 2.28% MVS-Synth (synthetic, GT depth; scene 0000 held out).
+  - "5700 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${mvs_synth_root}', dataset_type='mvs_synth', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=5700, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, exclude_sequence_ids=['0000']), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 4.52% TartanAir (static forward-moving scenes).
+  - "11300 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${tartanair_root}', dataset_type='tartanair', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=11300, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=1.0, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/torch_profiler/rank0.1780383278694422947.pt.trace.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c922ef6e320bae2b92df1353ebc22a2c12ff04da2f95db04157dd3d501f2a20
+size 4772489474

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/debug-internal.log

@@ -0,0 +1,87 @@
+{"time":"2026-06-02T06:50:49.974405605Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T06:50:49.976821599Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T06:50:50.36107604Z","level":"INFO","msg":"stream: created new stream","id":"nn58xrpu"}
+{"time":"2026-06-02T06:50:50.36114999Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T06:50:50.362005773Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T06:50:50.362019596Z","level":"INFO","msg":"writer: started","stream_id":"nn58xrpu"}
+{"time":"2026-06-02T06:50:50.362095786Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T06:50:51.690481716Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T06:50:51.888011116Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:01.410733865Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T06:51:01.428514695Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":168}
+{"time":"2026-06-02T06:51:06.691419747Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":122,"uploaded_len":2}
+{"time":"2026-06-02T06:51:06.9238499Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:21.69127157Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T06:51:21.913715771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:36.690922534Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-06-02T06:51:36.965752742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:51.692183121Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":123,"console_lines":1}
+{"time":"2026-06-02T06:51:51.980180422Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:06.691301178Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":124,"console_lines":2}
+{"time":"2026-06-02T06:52:07.06257193Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:21.69127678Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":2,"events_offset":10,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:52:22.085867072Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:36.69122883Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:52:36.896820176Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:51.691346553Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:52:51.892989007Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:06.691797368Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:06.871983169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:21.691461565Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:21.883845999Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:36.693338948Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:36.916559761Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:51.69176395Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":22,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:51.993609202Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:06.690851286Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:54:07.004048439Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:21.691620646Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-06-02T06:54:22.147645092Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:36.690891775Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-06-02T06:54:37.065124365Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:51.692454398Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T06:54:51.918197063Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:06.69053338Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T06:55:06.905887225Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:21.691518Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:55:21.947899715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:36.691140706Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:55:36.952030582Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:51.691789048Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":2,"events_offset":38,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:55:51.898682683Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:06.691691779Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:06.985943244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:21.696443399Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:21.906715321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:36.691377746Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":2,"events_offset":44,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:36.931509511Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:51.691747719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:51.960969812Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:06.691390544Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:07.050274115Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:21.69204273Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":50,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:22.186614525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:36.691431427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:36.925377253Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:51.691892104Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":54,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:51.935668512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:06.691226293Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":56,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:06.90327617Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:21.691228117Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:21.922420057Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:36.691373952Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:36.932965537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:51.691882754Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":2,"events_offset":62,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:51.913675918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:06.691802803Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:59:06.9232588Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:21.691448365Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:59:21.938579666Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:30.473233165Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T06:59:30.474469258Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":127,"console_lines":99,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T06:59:30.683502844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:30.684680347Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T06:59:30.689378199Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T06:59:30.689457461Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T06:59:30.690169153Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T06:59:30.690201056Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 06:50:49,349 INFO    MainThread:199010 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_setup.py:_flush():81] Configure stats pid to 199010
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/logs/debug.log
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/logs/debug-internal.log
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_init.py:init():848] calling init triggers
+2026-06-02 06:50:49,351 INFO    MainThread:199010 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 0, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 06:50:49,351 INFO    MainThread:199010 [wandb_init.py:init():896] starting backend
+2026-06-02 06:50:49,963 INFO    MainThread:199010 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 06:50:49,972 INFO    MainThread:199010 [wandb_init.py:init():919] backend started and connected
+2026-06-02 06:50:49,975 INFO    MainThread:199010 [wandb_init.py:init():989] updated telemetry
+2026-06-02 06:50:50,021 INFO    MainThread:199010 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 06:50:50,887 INFO    MainThread:199010 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 06:50:51,673 INFO    MainThread:199010 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 06:50:51,674 INFO    MainThread:199010 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 06:50:51,674 INFO    MainThread:199010 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 06:50:51,674 INFO    MainThread:199010 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 06:50:51,682 INFO    MainThread:199010 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 06:59:29,656 INFO    MainThread:199010 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/nn58xrpu
+2026-06-02 06:59:29,663 INFO    MainThread:199010 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 06:59:29,664 INFO    MainThread:199010 [wandb_run.py:_restore():2490] restore
+2026-06-02 06:59:29,664 INFO    MainThread:199010 [wandb_run.py:_restore():2496] restore done
+2026-06-02 06:59:30,688 INFO    MainThread:199010 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/files/config.yaml

@@ -0,0 +1,700 @@
+_wandb:
+    value:
+        cli_version: 0.26.0
+        e:
+            e80r6z0g9k261lnpba92qxhgslcr7ks2:
+                args:
+                    - --model_id_with_origin_paths
+                    - Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+                    - --tokenizer_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+                    - --task
+                    - sft:v2v
+                    - --model_variant
+                    - vt2v
+                    - --v2v_concat_mode
+                    - width
+                    - --height
+                    - "288"
+                    - --width
+                    - "512"
+                    - --num_frames
+                    - "49"
+                    - --track4d360_data_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+                    - --steps_per_epoch
+                    - "25000"
+                    - --num_epochs
+                    - "200"
+                    - --learning_rate
+                    - "2e-5"
+                    - --gradient_accumulation_steps
+                    - "1"
+                    - --save_steps
+                    - "500"
+                    - --remove_prefix_in_ckpt
+                    - pipe.dit.
+                    - --trainable_models
+                    - dit.domain_embedding,dit.control_patch_embedding
+                    - --dit_finetune_modules
+                    - self_attn
+                    - --track_injection_mode
+                    - per_block
+                    - --track_injection_block_mode
+                    - concat_project
+                    - --track_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml
+                    - --train_sigma_shift
+                    - "5.0"
+                    - --use_query_frame_impulse_condition
+                    - --use_plucker_camera_control
+                    - --enable_v2v_plucker_camera_control
+                    - --use_plucker_raymap_tokens
+                    - --use_perlayer_camera_control
+                    - --camera_injection_mode
+                    - projector
+                    - --camera_plucker_encoder_mode
+                    - patchify
+                    - --use_dense_branch
+                    - --dense_in_channels
+                    - "5"
+                    - --dense_proj_dim
+                    - "32"
+                    - --dense_num_residual_blocks
+                    - "0"
+                    - --dense_alpha_track
+                    - "1.0"
+                    - --dense_conditioning_noise_max_level
+                    - "0.15"
+                    - --source_mask_boundary_depth_dropout_prob
+                    - "0.5"
+                    - --source_mask_boundary_kernel_size
+                    - "5"
+                    - --dense_warp_mode
+                    - hybrid_softsplat
+                    - --dense_warp_motion_threshold
+                    - "0.005"
+                    - --cond_drop_geo_prob
+                    - "0.1"
+                    - --cond_drop_track_prob
+                    - "0.1"
+                    - --cond_drop_dense_prob
+                    - "0.0"
+                    - --initial_step
+                    - "2530"
+                    - --output_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout
+                    - --dataset_num_workers
+                    - "7"
+                    - --dataloader_prefetch_factor
+                    - "2"
+                    - --drop_text_prob
+                    - "0.1"
+                    - --debug_latents_every
+                    - "500"
+                    - --allow_existing_checkpoints
+                    - --wandb_project
+                    - track4d-360
+                    - --wandb_name
+                    - ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp
+                    - --wandb_tags
+                    - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+                    - --use_wandb
+                    - --vxf_init_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+                    - --use_gradient_checkpointing
+                    - --resume_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+                    - --max_train_steps
+                    - "2560"
+                    - --torch_profile_active_steps
+                    - "5"
+                    - --torch_profile_wait_steps
+                    - "3"
+                    - --torch_profile_warmup_steps
+                    - "1"
+                    - --torch_profile_repeat
+                    - "1"
+                    - --no_torch_profile_record_shapes
+                    - --no_torch_profile_memory
+                codePath: scripts/wan2.1_fun/train_track4d360.py
+                codePathLocal: scripts/wan2.1_fun/train_track4d360.py
+                cpu_count: 288
+                cpu_count_logical: 288
+                cudaVersion: "12.7"
+                disk:
+                    /:
+                        total: "255795724288"
+                        used: "526450688"
+                email: yushi.lan@outlook.com
+                executable: /projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12
+                git:
+                    commit: b6681ed2127310d14bc5b93ae5f1d20df8b9e907
+                    remote: git@github.com:NIRVANALAN/track4d-360.git
+                gpu: NVIDIA GH200 120GB
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-c78246be-bb85-931c-a8c3-f605c70f9873
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-d557559f-6b05-d9af-f672-98b04500c99e
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-fc063581-c253-951d-1735-1c26372c8d1e
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-dca24d57-11a6-3aa7-883b-6f639eb27927
+                host: nid010271
+                memory:
+                    total: "919613407232"
+                os: Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38
+                program: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py
+                python: CPython 3.12.13
+                root: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout
+                slurm:
+                    cluster_name: gracehopper
+                    conf: /var/spool/slurmd/conf-cache/slurm.conf
+                    cpu_bind: quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_list: 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_type: 'mask_cpu:'
+                    cpu_bind_verbose: quiet
+                    cpus_on_node: "288"
+                    distribution: cyclic
+                    gpus: "32"
+                    gpus_on_node: "4"
+                    gpus_per_node: "4"
+                    gtids: "0"
+                    job_account: brics.u6ew
+                    job_cpus_per_node: 288(x8)
+                    job_end_time: "1780389885"
+                    job_gid: "1483804847"
+                    job_gpus: 0,1,2,3
+                    job_id: "4982941"
+                    job_name: 14b_pair1x49_profile_ablate_denseres0_fsdp2
+                    job_nodelist: nid[010271,010275,010290,010292,010296-010299]
+                    job_num_nodes: "8"
+                    job_partition: workq
+                    job_qos: normal
+                    job_start_time: "1780382685"
+                    job_uid: "1483804847"
+                    job_user: yushi.u6ew
+                    jobid: "4982941"
+                    launch_node_ipaddr: 10.242.18.96
+                    localid: "0"
+                    mpi_type: pmi2
+                    nnodes: "8"
+                    nodeid: "0"
+                    nodelist: nid[010271,010275,010290,010292,010296-010299]
+                    nprocs: "8"
+                    ntasks: "8"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    srun_comm_host: 10.242.18.96
+                    srun_comm_port: "41477"
+                    step_gpus: 0,1,2,3
+                    step_id: "0"
+                    step_launcher_port: "41477"
+                    step_nodelist: nid[010271,010275,010290,010292,010296-010299]
+                    step_num_nodes: "8"
+                    step_num_tasks: "8"
+                    step_resv_ports: 26616-26617
+                    step_tasks_per_node: 1(x8)
+                    stepid: "0"
+                    submit_dir: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360
+                    submit_host: login40
+                    task_pid: "198920"
+                    tasks_per_node: 1(x8)
+                    topology_addr: global.group4.nid010271
+                    topology_addr_pattern: switch.switch.node
+                    umask: "0022"
+                startedAt: "2026-06-02T06:50:49.340519Z"
+                writerId: e80r6z0g9k261lnpba92qxhgslcr7ks2
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 98
+                - 106
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+                - 106
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 41
+                - 61
+            "4": 3.12.13
+            "5": 0.26.0
+            "6": 4.46.2
+            "12": 0.26.0
+            "13": linux-aarch64
+allow_existing_checkpoints:
+    value: true
+camera_condition_cfg_dropout_prob:
+    value: 0
+camera_injection_mode:
+    value: projector
+camera_plucker_encoder_mode:
+    value: patchify
+channel_concat_stage:
+    value: null
+cond_drop_dense_prob:
+    value: 0
+cond_drop_geo_prob:
+    value: 0.1
+cond_drop_track_prob:
+    value: 0.1
+data_file_keys:
+    value: image,video
+dataloader_prefetch_factor:
+    value: 2
+dataset_base_path:
+    value: null
+dataset_metadata_path:
+    value: null
+dataset_num_workers:
+    value: 7
+dataset_repeat:
+    value: 1
+debug_latents:
+    value: false
+debug_latents_every:
+    value: 500
+debug_latents_fps:
+    value: 16
+debug_latents_max_frames:
+    value: null
+debug_latents_outdir:
+    value: debug_latents
+debug_latents_quality:
+    value: 5
+debug_latents_vae_tile_size:
+    value:
+        - 30
+        - 52
+debug_latents_vae_tile_stride:
+    value:
+        - 15
+        - 26
+debug_latents_vae_tiled:
+    value: true
+debug_missing_grads:
+    value: false
+dense_alpha_track:
+    value: 1
+dense_conditioning_noise_max_level:
+    value: 0.15
+dense_hybrid_disable_dynamic_masks:
+    value: false
+dense_in_channels:
+    value: 5
+dense_num_residual_blocks:
+    value: 0
+dense_only_condition_modules:
+    value: false
+dense_proj_dim:
+    value: 32
+dense_warp_mode:
+    value: hybrid_softsplat
+dense_warp_motion_threshold:
+    value: 0.005
+disable_track_modules:
+    value: false
+dit_finetune_modules:
+    value: self_attn
+drop_i2v_prob:
+    value: null
+drop_t2v_prob:
+    value: null
+drop_text_prob:
+    value: 0.1
+drop_video_prob:
+    value: 0
+dummy:
+    value: false
+enable_v2v_plucker_camera_control:
+    value: true
+explicit_motion_loss_decode_checkpoint:
+    value: false
+explicit_motion_loss_decode_chunk_size:
+    value: 0
+explicit_motion_loss_snr_sigmoid:
+    value: true
+explicit_motion_loss_snr_sigmoid_a:
+    value: 1
+explicit_motion_loss_snr_sigmoid_b:
+    value: -5
+explicit_motion_loss_snr_weight:
+    value: true
+explicit_motion_loss_weight:
+    value: 0
+extra_inputs:
+    value: null
+find_unused_parameters:
+    value: false
+fixed_num_inference_steps:
+    value: 50
+fixed_sigma:
+    value: null
+fixed_step_frac:
+    value: null
+fp8_models:
+    value: null
+freeze_lora_after_load:
+    value: false
+freeze_track_block_injector:
+    value: false
+freeze_track4d_condition_modules:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gs_abs_depth_loss_weight:
+    value: 0
+gs_assoc_track_2d_loss_weight:
+    value: 0
+gs_depth_loss_weight:
+    value: 0.05
+gs_direct_pos_loss_weight:
+    value: 0
+gs_disable_delta_opacity:
+    value: false
+gs_dyn_depth_loss_weight:
+    value: 0.5
+gs_dynamic_max_t_video:
+    value: 64
+gs_dynamic_mode:
+    value: static
+gs_feat_dim:
+    value: 1024
+gs_head_init_checkpoint:
+    value: null
+gs_head_style:
+    value: flashworld
+gs_legacy_union_render:
+    value: false
+gs_loss_weight:
+    value: 1
+gs_lpips_loss_weight:
+    value: 0.5
+gs_lyra_depth_param:
+    value: exp
+gs_lyra_dfar:
+    value: 10
+gs_lyra_dnear:
+    value: 0.1
+gs_lyra_log_depth_shift:
+    value: 0
+gs_mse_loss_weight:
+    value: 1
+gs_num_points_per_pixel:
+    value: 2
+gs_only_training:
+    value: false
+gs_opa_tv_weight:
+    value: 0.01
+gs_opacity_loss_weight:
+    value: 0.1
+gs_opacity_prune_keep_frac:
+    value: 1
+gs_pos_loss_weight:
+    value: 1
+gs_predict_rotation:
+    value: false
+gs_render_patch_size:
+    value: null
+gs_replay_cache_size:
+    value: 0
+gs_rgb_loss_weight:
+    value: null
+gs_scale_cap:
+    value: null
+gs_sh_degree:
+    value: 2
+gs_track_accel_loss_weight:
+    value: 0
+gs_track_canonical_detach:
+    value: false
+gs_track_canonical_loss_weight:
+    value: 0
+gs_track_delta_loss_weight:
+    value: 0
+gs_track_final_pos_loss_weight:
+    value: 0
+gs_track_joint_render:
+    value: false
+gs_track_loss_quantile:
+    value: 0.98
+gs_track_supervision_mode:
+    value: pointmap
+gs_xyz_tv_weight:
+    value: 0.01
+gs_zero_features:
+    value: false
+height:
+    value: 288
+initial_step:
+    value: 2530
+initialize_model_on_cpu:
+    value: false
+input_video_noise_random_max:
+    value: 500
+input_video_noise_strength:
+    value: 0
+joint_depth_prediction:
+    value: false
+joint_track_depth_prediction:
+    value: false
+joint_track_prediction:
+    value: false
+learning_rate:
+    value: 2e-05
+lora_alpha:
+    value: null
+lora_base_model:
+    value: null
+lora_checkpoint:
+    value: null
+lora_rank:
+    value: 32
+lora_target_modules:
+    value: q,k,v,o,ffn.0,ffn.2
+max_checkpoints_to_keep:
+    value: 5
+max_pixels:
+    value: 1048576
+max_train_steps:
+    value: 2560
+model_id_with_origin_paths:
+    value: Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+model_paths:
+    value: null
+model_variant:
+    value: vt2v
+multi_frame_motion_threshold:
+    value: 0.01
+multi_frame_warped_appearance:
+    value: false
+no_gs_latent_detach:
+    value: false
+normalize_track_with_xyz_cond_stats:
+    value: false
+num_epochs:
+    value: 200
+num_frames:
+    value: 49
+num_workers:
+    value: 7
+offload_models:
+    value: null
+output_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout
+plucker_use_ray_origin:
+    value: false
+pointmap_stats_path:
+    value: null
+predict_track_residual:
+    value: false
+prefetch_factor:
+    value: 2
+prenormalized_track_video_signed_log:
+    value: false
+preset_lora_model:
+    value: null
+preset_lora_path:
+    value: null
+remove_prefix_in_ckpt:
+    value: pipe.dit.
+residual_linear_normalization:
+    value: false
+residual_log_scale_factor:
+    value: 3
+residual_target_xyz_source:
+    value: clean
+residual_zero_center_normalization:
+    value: false
+resume_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+save_every_n_epochs:
+    value: 2
+save_steps:
+    value: 500
+scene:
+    value: null
+sequence_idx:
+    value: null
+skip_diffusion_loss:
+    value: false
+skip_v2v_required_trainables:
+    value: false
+source_mask_boundary_depth_dropout_prob:
+    value: 0.5
+source_mask_boundary_kernel_size:
+    value: 5
+steps_per_epoch:
+    value: 25000
+stream3r_common_scale_stat:
+    value: mean
+syn4d_root:
+    value: /scratch/shared/beegfs/zeren/Syn4D
+target_depth_log_cap:
+    value: 14
+target_depth_max_disparity:
+    value: 5
+target_depth_norm_type:
+    value: log_depth
+task:
+    value: sft:v2v
+tokenizer_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+torch_distributed_debug:
+    value: null
+torch_profile_active_steps:
+    value: 5
+torch_profile_dir:
+    value: null
+torch_profile_memory:
+    value: false
+torch_profile_record_shapes:
+    value: false
+torch_profile_repeat:
+    value: 1
+torch_profile_wait_steps:
+    value: 3
+torch_profile_warmup_steps:
+    value: 1
+torch_profile_with_stack:
+    value: false
+track_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml
+track_injection_block_mode:
+    value: concat_project
+track_injection_mode:
+    value: per_block
+track_spatial_stride:
+    value: 1
+track_xyz_normalization:
+    value: minmax
+track_xyz_quantile_high:
+    value: 0.98
+track_xyz_quantile_low:
+    value: 0.02
+track4d360_data_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+train_plucker_camera_control:
+    value: false
+train_sigma_shift:
+    value: 5
+trainable_models:
+    value: dit.domain_embedding,dit.control_patch_embedding
+use_dense_branch:
+    value: true
+use_dense_null_token_fallback:
+    value: false
+use_depth_null_token:
+    value: false
+use_gen3c_warp_latent_channels:
+    value: false
+use_gradient_checkpointing:
+    value: true
+use_gradient_checkpointing_offload:
+    value: false
+use_gs_feedback_training:
+    value: false
+use_gs_output:
+    value: false
+use_perlayer_camera_control:
+    value: true
+use_plucker_camera_control:
+    value: true
+use_plucker_raymap_tokens:
+    value: true
+use_query_frame_impulse_condition:
+    value: true
+use_residual_tracks:
+    value: true
+use_track_null_token_fallback:
+    value: false
+use_vista4d_latent_streams:
+    value: false
+use_wandb:
+    value: true
+use_xyz_video_condition:
+    value: false
+v2v_channel_patch_embed_mode:
+    value: add
+v2v_concat_mode:
+    value: width
+vxf_init_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+wandb_entity:
+    value: null
+wandb_log_freq:
+    value: 100
+wandb_log_gradients:
+    value: false
+wandb_log_parameters:
+    value: false
+wandb_name:
+    value: ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp
+wandb_notes:
+    value: null
+wandb_project:
+    value: track4d-360
+wandb_tags:
+    value:
+        - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+wandb_watch_model:
+    value: false
+warped_appearance_fusion:
+    value: concat_proj
+warped_condition_mode:
+    value: "off"
+warped_geom_only_dense:
+    value: true
+weight_decay:
+    value: 0.01
+width:
+    value: 512
+xyz_video_condition_aug_prob:
+    value: 0
+xyz_video_condition_cfg_dropout_prob:
+    value: 0
+xyz_video_condition_dropout_prob:
+    value: 0
+xyz_video_condition_frame_scale_std:
+    value: 0
+xyz_video_condition_mode:
+    value: xyz
+xyz_video_condition_noise_std:
+    value: 0
+xyz_video_condition_scale:
+    value: 1
+xyz_video_condition_z_noise_std:
+    value: 0

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/files/output.log

@@ -0,0 +1,226 @@
+✓ WandB initialized: https://wandb.ai/nirvanalan/track4d-360/runs/nn58xrpu
+/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/lib/python3.12/site-packages/accelerate/utils/fsdp_utils.py:742: UserWarning: FSDP upcast of low precision parameters to fp32 (since mixed_precision != 'no') may affect the precision of model checkpoints. This effects 589 parameters: ['pipe.dit.domain_embedding', 'pipe.dit.blocks.0.self_attn.q.weight', 'pipe.dit.blocks.0.self_attn.q.bias', 'pipe.dit.blocks.0.self_attn.k.weight', 'pipe.dit.blocks.0.self_attn.k.bias', 'pipe.dit.blocks.0.self_attn.v.weight', 'pipe.dit.blocks.0.self_attn.v.bias', 'pipe.dit.blocks.0.self_attn.o.weight', 'pipe.dit.blocks.0.self_attn.o.bias', 'pipe.dit.blocks.0.self_attn.norm_q.weight', 'pipe.dit.blocks.0.self_attn.norm_k.weight', 'pipe.dit.blocks.0.camera_projector.weight', 'pipe.dit.blocks.0.camera_projector.bias', 'pipe.dit.blocks.1.self_attn.q.weight', 'pipe.dit.blocks.1.self_attn.q.bias', 'pipe.dit.blocks.1.self_attn.k.weight', 'pipe.dit.blocks.1.self_attn.k.bias', 'pipe.dit.blocks.1.self_attn.v.weight', 'pipe.dit.blocks.1.self_attn.v.bias', 'pipe.dit.blocks.1.self_attn.o.weight', 'pipe.dit.blocks.1.self_attn.o.bias', 'pipe.dit.blocks.1.self_attn.norm_q.weight', 'pipe.dit.blocks.1.self_attn.norm_k.weight', 'pipe.dit.blocks.1.camera_projector.weight', 'pipe.dit.blocks.1.camera_projector.bias', 'pipe.dit.blocks.2.self_attn.q.weight', 'pipe.dit.blocks.2.self_attn.q.bias', 'pipe.dit.blocks.2.self_attn.k.weight', 'pipe.dit.blocks.2.self_attn.k.bias', 'pipe.dit.blocks.2.self_attn.v.weight', 'pipe.dit.blocks.2.self_attn.v.bias', 'pipe.dit.blocks.2.self_attn.o.weight', 'pipe.dit.blocks.2.self_attn.o.bias', 'pipe.dit.blocks.2.self_attn.norm_q.weight', 'pipe.dit.blocks.2.self_attn.norm_k.weight', 'pipe.dit.blocks.2.camera_projector.weight', 'pipe.dit.blocks.2.camera_projector.bias', 'pipe.dit.blocks.3.self_attn.q.weight', 'pipe.dit.blocks.3.self_attn.q.bias', 'pipe.dit.blocks.3.self_attn.k.weight', 'pipe.dit.blocks.3.self_attn.k.bias', 'pipe.dit.blocks.3.self_attn.v.weight', 'pipe.dit.blocks.3.self_attn.v.bias', 'pipe.dit.blocks.3.self_attn.o.weight', 'pipe.dit.blocks.3.self_attn.o.bias', 'pipe.dit.blocks.3.self_attn.norm_q.weight', 'pipe.dit.blocks.3.self_attn.norm_k.weight', 'pipe.dit.blocks.3.camera_projector.weight', 'pipe.dit.blocks.3.camera_projector.bias', 'pipe.dit.blocks.4.self_attn.q.weight', 'pipe.dit.blocks.4.self_attn.q.bias', 'pipe.dit.blocks.4.self_attn.k.weight', 'pipe.dit.blocks.4.self_attn.k.bias', 'pipe.dit.blocks.4.self_attn.v.weight', 'pipe.dit.blocks.4.self_attn.v.bias', 'pipe.dit.blocks.4.self_attn.o.weight', 'pipe.dit.blocks.4.self_attn.o.bias', 'pipe.dit.blocks.4.self_attn.norm_q.weight', 'pipe.dit.blocks.4.self_attn.norm_k.weight', 'pipe.dit.blocks.4.camera_projector.weight', 'pipe.dit.blocks.4.camera_projector.bias', 'pipe.dit.blocks.5.self_attn.q.weight', 'pipe.dit.blocks.5.self_attn.q.bias', 'pipe.dit.blocks.5.self_attn.k.weight', 'pipe.dit.blocks.5.self_attn.k.bias', 'pipe.dit.blocks.5.self_attn.v.weight', 'pipe.dit.blocks.5.self_attn.v.bias', 'pipe.dit.blocks.5.self_attn.o.weight', 'pipe.dit.blocks.5.self_attn.o.bias', 'pipe.dit.blocks.5.self_attn.norm_q.weight', 'pipe.dit.blocks.5.self_attn.norm_k.weight', 'pipe.dit.blocks.5.camera_projector.weight', 'pipe.dit.blocks.5.camera_projector.bias', 'pipe.dit.blocks.6.self_attn.q.weight', 'pipe.dit.blocks.6.self_attn.q.bias', 'pipe.dit.blocks.6.self_attn.k.weight', 'pipe.dit.blocks.6.self_attn.k.bias', 'pipe.dit.blocks.6.self_attn.v.weight', 'pipe.dit.blocks.6.self_attn.v.bias', 'pipe.dit.blocks.6.self_attn.o.weight', 'pipe.dit.blocks.6.self_attn.o.bias', 'pipe.dit.blocks.6.self_attn.norm_q.weight', 'pipe.dit.blocks.6.self_attn.norm_k.weight', 'pipe.dit.blocks.6.camera_projector.weight', 'pipe.dit.blocks.6.camera_projector.bias', 'pipe.dit.blocks.7.self_attn.q.weight', 'pipe.dit.blocks.7.self_attn.q.bias', 'pipe.dit.blocks.7.self_attn.k.weight', 'pipe.dit.blocks.7.self_attn.k.bias', 'pipe.dit.blocks.7.self_attn.v.weight', 'pipe.dit.blocks.7.self_attn.v.bias', 'pipe.dit.blocks.7.self_attn.o.weight', 'pipe.dit.blocks.7.self_attn.o.bias', 'pipe.dit.block
+  warnings.warn(
+
+================================================================================
+TRAINABLE PARAMETERS VERIFICATION
+================================================================================
+
+Trainable modules (94 module groups):
+
+  📦 pipe.dit.blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+     pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.dit._camera_patchify_conv:
+     Total params: 36,870
+         ├─ weight: 36,864
+         ├─ bias: 6
+
+  📦 pipe.dit._camera_plucker_patchify_encoder:
+     Total params: 35,840
+         ├─ weight: 30,720
+         ├─ bias: 5,120
+
+  📦 pipe.dit.control_patch_embedding:
+     Total params: 332,800
+         ├─ weight: 327,680
+         ├─ bias: 5,120
+
+  📦 pipe.dit.domain_embedding:
+     Total params: 10,240
+     pipe.dit.domain_embedding: 10,240
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+     pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_adapter.residual_blocks:
+     Total params: 55,360
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         ├─ 16.weight: 27,013,120
+         ├─ 17.weight: 27,013,120
+         ├─ 18.weight: 27,013,120
+         ├─ 19.weight: 27,013,120
+         ├─ 2.weight: 27,013,120
+         ├─ 20.weight: 27,013,120
+         ├─ 21.weight: 27,013,120
+         ├─ 22.weight: 27,013,120
+         ├─ 23.weight: 27,013,120
+         ├─ 24.weight: 27,013,120
+         ├─ 25.weight: 27,013,120
+         ├─ 26.weight: 27,013,120
+         ├─ 27.weight: 27,013,120
+         ├─ 28.weight: 27,013,120
+         ├─ 29.weight: 27,013,120
+         ├─ 3.weight: 27,013,120
+         ├─ 30.weight: 27,013,120
+         ├─ … (55 more)
+
+────────────────────────────────────────────────────────────────────────────────
+NOTE: FSDP is active; counts below are from the rank-local post-wrap parameter view, not a full global parameter census.
+Total trainable parameters: 6,325,685,810 (6325.69M)
+Total frozen parameters:    10,363,571,200 (10363.57M)
+Percentage trainable (overall): 37.90%
+Percentage trainable (in requested modules: dit): 33.60%
+================================================================================
+
+[torch-profiler] rank0 enabled (wait=3, warmup=1, active=5, repeat=1) -> /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/torch_profiler
+Epoch 1/200:   0%|          | 29/7813 [08:13<36:46:56, 17.01s/it, opt_step=2559, loss=0.0681]  
+[Track4D-360] First train batch runtime flags: use_gradient_checkpointing=True, use_gradient_checkpointing_offload=False
+[track4d-runtime] rank0 patched DiTBlock.forward saw mode=track block_idx=0
+[Attention] Using: Flash Attention 2 (flash_attn)
+[soft_splat_warp] WARN: tracks grid 160x294 (N=47040) != image grid 288x512 (N=147456); bilinear-resampling tracks_world to image grid. This warning fires once per process.
+
+[Training] Reached max_train_steps=2560. Stopping training.
+
+[Checkpoint] Saving 589 parameters (total 6,325,685,810 ≈ 6325.69M):
+
+  📦 blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 _camera_patchify_conv.bias:
+     Total params: 6
+         _camera_patchify_conv.bias: 6
+
+  📦 _camera_patchify_conv.weight:
+     Total params: 36,864
+         _camera_patchify_conv.weight: 36,864
+
+  📦 _camera_plucker_patchify_encoder.bias:
+     Total params: 5,120
+         _camera_plucker_patchify_encoder.bias: 5,120
+
+  📦 _camera_plucker_patchify_encoder.weight:
+     Total params: 30,720
+         _camera_plucker_patchify_encoder.weight: 30,720
+
+  📦 control_patch_embedding.bias:
+     Total params: 5,120
+         control_patch_embedding.bias: 5,120
+
+  📦 control_patch_embedding.weight:
+     Total params: 327,680
+         control_patch_embedding.weight: 327,680
+
+  📦 domain_embedding:
+     Total params: 10,240
+         domain_embedding: 10,240
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+         pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+         pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_adapter.residual_blocks:
+     Total params: 55,360
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         … (72 more)

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/files/requirements.txt

@@ -0,0 +1,208 @@
+videox-fun==1.0.1
+track4d-360==0.1.0
+func_timeout==4.3.5
+click==8.3.2
+pip==26.0.1
+nvidia-cuda-nvrtc-cu12==12.6.77
+ImageIO==2.37.3
+numpy==1.26.4
+boto3==1.42.91
+albucore==0.0.24
+lpips==0.1.4
+torchsde==0.2.6
+tomesd==0.1.3
+yarl==1.23.0
+typer==0.24.1
+triton==3.5.0
+cycler==0.12.1
+multidict==6.7.1
+OpenEXR==3.4.10
+soxr==1.0.0
+kiwisolver==1.5.0
+simsimd==6.5.16
+parso==0.8.6
+einops==0.8.2
+ftfy==6.3.1
+platformdirs==4.9.6
+groovy==0.1.2
+nvidia-nvtx-cu12==12.6.77
+sentry-sdk==2.58.0
+certifi==2026.2.25
+markdown-it-py==4.0.0
+uvicorn==0.38.0
+nvidia-cudnn-cu12==9.10.2.21
+natsort==8.4.0
+awscrt==0.31.2
+accelerate==1.13.0
+deepspeed==0.18.9
+matplotlib==3.10.8
+mdurl==0.1.2
+regex==2026.4.4
+nvidia-cuda-cupti-cu12==12.6.80
+shellingham==1.5.4
+joblib==1.5.3
+zipp==3.23.1
+ipython_pygments_lexers==1.1.1
+executing==2.2.1
+safetensors==0.7.0
+typing-inspection==0.4.2
+httpcore==1.0.9
+semantic-version==2.10.0
+wandb==0.26.0
+asttokens==3.0.1
+datasets==4.8.4
+decorator==5.2.1
+attrs==26.1.0
+six==1.17.0
+importlib_metadata==9.0.0
+py-cpuinfo==9.0.0
+requests==2.33.1
+xxhash==3.6.0
+pytorch-lightning==2.6.1
+fsspec==2026.2.0
+albumentations==2.0.8
+soundfile==0.13.1
+torchaudio==2.9.0
+torchmetrics==1.9.0
+nvidia-cusolver-cu12==11.7.1.2
+scipy==1.17.1
+jmespath==1.1.0
+threadpoolctl==3.6.0
+plyfile==1.1.2
+peft==0.19.1
+pycparser==3.0
+Markdown==3.10.2
+aiosignal==1.4.0
+pydantic==2.13.2
+decord2==3.3.0
+safehttpx==0.1.7
+scikit-image==0.26.0
+fonttools==4.62.1
+lazy-loader==0.5
+smmap==5.0.3
+starlette==1.0.0
+Werkzeug==3.1.8
+propcache==0.4.1
+tensorboard==2.20.0
+rich==15.0.0
+modelscope==1.36.3
+absl-py==2.4.0
+traitlets==5.14.3
+networkx==3.6.1
+hf-xet==1.4.3
+h5py==3.16.0
+PyYAML==6.0.3
+Jinja2==3.1.6
+nvidia-cusparse-cu12==12.5.4.2
+idna==3.11
+transformers==4.46.2
+httpx==0.28.1
+MarkupSafe==3.0.3
+tensorboard-data-server==0.7.2
+pooch==1.9.0
+torch==2.9.0+cu126
+soupsieve==2.8.3
+numba==0.65.0
+torchdiffeq==0.2.5
+jedi==0.19.2
+charset-normalizer==3.4.7
+annotated-doc==0.0.4
+gradio==6.12.0
+Pygments==2.20.0
+pandas==3.0.2
+stack-data==0.6.3
+contourpy==1.3.3
+tomlkit==0.14.0
+scikit-learn==1.8.0
+librosa==0.11.0
+nvidia-cusparselt-cu12==0.7.1
+pytz==2026.1.post1
+go-with-the-track==0.1.0
+yunchang==0.6.4
+onnxruntime==1.24.4
+ninja==1.13.0
+sentencepiece==0.2.1
+pexpect==4.9.0
+multiprocess==0.70.19
+omegaconf==2.3.0
+h11==0.16.0
+pydub==0.25.1
+nvidia-cufft-cu12==11.3.0.4
+xfuser==0.4.5
+tifffile==2026.3.3
+msgpack==1.1.2
+psutil==7.2.2
+ipython==9.12.0
+hf-gradio==0.4.1
+diffusers==0.33.1
+python-dateutil==2.9.0.post0
+orjson==3.11.8
+huggingface_hub==0.36.2
+nvidia-cuda-runtime-cu12==12.6.77
+beautifulsoup4==4.14.3
+ptyprocess==0.7.0
+protobuf==7.34.1
+GitPython==3.1.46
+sympy==1.14.0
+packaging==26.1
+aiohttp==3.13.5
+nvidia-nvshmem-cu12==3.3.20
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+gradio_client==2.4.1
+stringzilla==4.6.0
+flatbuffers==25.12.19
+filelock==3.28.0
+imageio-ffmpeg==0.6.0
+fastapi==0.136.0
+nvidia-nvjitlink-cu12==12.6.85
+s3transfer==0.16.0
+python-multipart==0.0.26
+DistVAE==0.0.0b5
+urllib3==2.6.3
+nvidia-nccl-cu12==2.27.5
+av==15.1.0
+pure_eval==0.2.3
+wheel==0.46.3
+dill==0.4.1
+flash_attn==2.8.3
+track4d-360==0.1.0
+opencv-python==4.11.0.86
+anyio==4.13.0
+pytest==9.0.3
+nvidia-cufile-cu12==1.11.1.6
+lightning-utilities==0.15.3
+pluggy==1.6.0
+grpcio==1.80.0
+cffi==2.0.0
+iniconfig==2.3.0
+videox-fun==1.0.1
+matplotlib-inline==0.2.1
+tokenizers==0.20.3
+pydantic_core==2.46.2
+aiohappyeyeballs==2.6.1
+typing_extensions==4.15.0
+timm==1.0.26
+pyarrow==23.0.1
+mpmath==1.3.0
+prompt_toolkit==3.0.52
+mediapy==1.2.6
+llvmlite==0.47.0
+opencv-python-headless==4.11.0.86
+hjson==3.1.0
+tqdm==4.67.3
+torchvision==0.24.0
+pyparsing==3.3.2
+nvidia-cublas-cu12==12.6.4.1
+botocore==1.42.91
+nanobind==2.9.2
+wcwidth==0.6.0
+gitdb==4.0.12
+frozenlist==1.8.0
+brotli==1.2.0
+trampoline==0.1.2
+pillow==12.2.0
+audioread==3.1.0
+lightning==2.6.1
+setuptools==82.0.1
+nvidia-curand-cu12==10.3.7.77

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/files/wandb-metadata.json

@@ -0,0 +1,235 @@
+{
+  "os": "Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-06-02T06:50:49.340519Z",
+  "args": [
+    "--model_id_with_origin_paths",
+    "Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth",
+    "--tokenizer_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl",
+    "--task",
+    "sft:v2v",
+    "--model_variant",
+    "vt2v",
+    "--v2v_concat_mode",
+    "width",
+    "--height",
+    "288",
+    "--width",
+    "512",
+    "--num_frames",
+    "49",
+    "--track4d360_data_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml",
+    "--steps_per_epoch",
+    "25000",
+    "--num_epochs",
+    "200",
+    "--learning_rate",
+    "2e-5",
+    "--gradient_accumulation_steps",
+    "1",
+    "--save_steps",
+    "500",
+    "--remove_prefix_in_ckpt",
+    "pipe.dit.",
+    "--trainable_models",
+    "dit.domain_embedding,dit.control_patch_embedding",
+    "--dit_finetune_modules",
+    "self_attn",
+    "--track_injection_mode",
+    "per_block",
+    "--track_injection_block_mode",
+    "concat_project",
+    "--track_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml",
+    "--train_sigma_shift",
+    "5.0",
+    "--use_query_frame_impulse_condition",
+    "--use_plucker_camera_control",
+    "--enable_v2v_plucker_camera_control",
+    "--use_plucker_raymap_tokens",
+    "--use_perlayer_camera_control",
+    "--camera_injection_mode",
+    "projector",
+    "--camera_plucker_encoder_mode",
+    "patchify",
+    "--use_dense_branch",
+    "--dense_in_channels",
+    "5",
+    "--dense_proj_dim",
+    "32",
+    "--dense_num_residual_blocks",
+    "0",
+    "--dense_alpha_track",
+    "1.0",
+    "--dense_conditioning_noise_max_level",
+    "0.15",
+    "--source_mask_boundary_depth_dropout_prob",
+    "0.5",
+    "--source_mask_boundary_kernel_size",
+    "5",
+    "--dense_warp_mode",
+    "hybrid_softsplat",
+    "--dense_warp_motion_threshold",
+    "0.005",
+    "--cond_drop_geo_prob",
+    "0.1",
+    "--cond_drop_track_prob",
+    "0.1",
+    "--cond_drop_dense_prob",
+    "0.0",
+    "--initial_step",
+    "2530",
+    "--output_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout",
+    "--dataset_num_workers",
+    "7",
+    "--dataloader_prefetch_factor",
+    "2",
+    "--drop_text_prob",
+    "0.1",
+    "--debug_latents_every",
+    "500",
+    "--allow_existing_checkpoints",
+    "--wandb_project",
+    "track4d-360",
+    "--wandb_name",
+    "ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp",
+    "--wandb_tags",
+    "ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug",
+    "--use_wandb",
+    "--vxf_init_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors",
+    "--use_gradient_checkpointing",
+    "--resume_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors",
+    "--max_train_steps",
+    "2560",
+    "--torch_profile_active_steps",
+    "5",
+    "--torch_profile_wait_steps",
+    "3",
+    "--torch_profile_warmup_steps",
+    "1",
+    "--torch_profile_repeat",
+    "1",
+    "--no_torch_profile_record_shapes",
+    "--no_torch_profile_memory"
+  ],
+  "program": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py",
+  "codePath": "scripts/wan2.1_fun/train_track4d360.py",
+  "codePathLocal": "scripts/wan2.1_fun/train_track4d360.py",
+  "git": {
+    "remote": "git@github.com:NIRVANALAN/track4d-360.git",
+    "commit": "b6681ed2127310d14bc5b93ae5f1d20df8b9e907"
+  },
+  "email": "yushi.lan@outlook.com",
+  "root": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout",
+  "host": "nid010271",
+  "executable": "/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12",
+  "cpu_count": 288,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA GH200 120GB",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "255795724288",
+      "used": "526450688"
+    }
+  },
+  "memory": {
+    "total": "919613407232"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-c78246be-bb85-931c-a8c3-f605c70f9873"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-d557559f-6b05-d9af-f672-98b04500c99e"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-fc063581-c253-951d-1735-1c26372c8d1e"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-dca24d57-11a6-3aa7-883b-6f639eb27927"
+    }
+  ],
+  "cudaVersion": "12.7",
+  "slurm": {
+    "cluster_name": "gracehopper",
+    "conf": "/var/spool/slurmd/conf-cache/slurm.conf",
+    "cpu_bind": "quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_list": "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_type": "mask_cpu:",
+    "cpu_bind_verbose": "quiet",
+    "cpus_on_node": "288",
+    "distribution": "cyclic",
+    "gpus": "32",
+    "gpus_on_node": "4",
+    "gpus_per_node": "4",
+    "gtids": "0",
+    "job_account": "brics.u6ew",
+    "job_cpus_per_node": "288(x8)",
+    "job_end_time": "1780389885",
+    "job_gid": "1483804847",
+    "job_gpus": "0,1,2,3",
+    "job_id": "4982941",
+    "job_name": "14b_pair1x49_profile_ablate_denseres0_fsdp2",
+    "job_nodelist": "nid[010271,010275,010290,010292,010296-010299]",
+    "job_num_nodes": "8",
+    "job_partition": "workq",
+    "job_qos": "normal",
+    "job_start_time": "1780382685",
+    "job_uid": "1483804847",
+    "job_user": "yushi.u6ew",
+    "jobid": "4982941",
+    "launch_node_ipaddr": "10.242.18.96",
+    "localid": "0",
+    "mpi_type": "pmi2",
+    "nnodes": "8",
+    "nodeid": "0",
+    "nodelist": "nid[010271,010275,010290,010292,010296-010299]",
+    "nprocs": "8",
+    "ntasks": "8",
+    "ntasks_per_node": "1",
+    "oom_kill_step": "0",
+    "prio_process": "0",
+    "procid": "0",
+    "srun_comm_host": "10.242.18.96",
+    "srun_comm_port": "41477",
+    "step_gpus": "0,1,2,3",
+    "step_id": "0",
+    "step_launcher_port": "41477",
+    "step_nodelist": "nid[010271,010275,010290,010292,010296-010299]",
+    "step_num_nodes": "8",
+    "step_num_tasks": "8",
+    "step_resv_ports": "26616-26617",
+    "step_tasks_per_node": "1(x8)",
+    "stepid": "0",
+    "submit_dir": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360",
+    "submit_host": "login40",
+    "task_pid": "198920",
+    "tasks_per_node": "1(x8)",
+    "topology_addr": "global.group4.nid010271",
+    "topology_addr_pattern": "switch.switch.node",
+    "umask": "0022"
+  },
+  "writerId": "e80r6z0g9k261lnpba92qxhgslcr7ks2"
+}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/track/injector/block_08/weight_norm":71.77714538574219,"train/hbm/post_t5_allocated_gb":28.869243621826172,"train/track/injector/block_34/weight_norm":71.64431762695312,"train/track/injector/block_01/grad_norm":0.0075633530505001545,"train/track/injector/block_20/grad_norm":0.003714467864483595,"train/track/injector/block_03/weight_norm":71.8019027709961,"train/track/injector/block_16/weight_norm":71.75887298583984,"train/loss":0.06805133819580078,"train/timing/step_total_s":12.027139294994413,"train/track/injector/block_24/weight_norm":71.72344207763672,"train/timing/backward_s":5.079326951003168,"train/track/injector/block_02/weight_norm":71.77030944824219,"train/track/injector/block_25/weight_norm":71.73070526123047,"train/track/injector/block_08/grad_norm":0.007247989531606436,"train/timing/unit/WanVideoUnit_TeaCache_s":2.044346729235258e-05,"train/timing/unit/WanVideoUnit_UnifiedSequenceParallel_s":2.825628143909853e-05,"train/track/injector/block_37/weight_norm":71.64723205566406,"train/track/injector/block_27/grad_norm":0.002658056328073144,"train/track/injector/block_32/grad_norm":0.002075374824926257,"train/track/injector/block_12/grad_norm":0.005726680625230074,"train/track/injector/block_30/grad_norm":0.002298362785950303,"_timestamp":1.7803835547502189e+09,"train/track/injector/block_39/grad_norm":0.004338877275586128,"train/track/injector/block_16/grad_norm":0.005405740346759558,"train/track/injector/block_36/grad_norm":0.0018088072538375854,"train/cond_dropout/drop_tracks":0.21875,"train/track/injector/block_03/grad_norm":0.007106975186616182,"train/timing/unit/WanVideoUnit_WarpedAppearance_s":6.011142613715492e-05,"train/track/injector/block_22/grad_norm":0.0046297223307192326,"train/track/injector/weight_norm_mean":71.72119140625,"train/track/injector/block_30/weight_norm":71.7083740234375,"train/hbm/allocated_gb":3.435391426086426,"train/hbm/post_t5_reserved_gb":28.94489097595215,"train/timing/unit/WanVideoUnit_CfgMerger_s":1.2710699593299069e-05,"train/timing/data_wait_s":0.0034507829986978322,"train/timing/optimizer_s":0.03327711300516967,"train/track/injector/block_32/weight_norm":71.66675567626953,"train/track/injector/block_25/grad_norm":0.0028126623947173357,"train/track/injector/block_20/weight_norm":71.7135009765625,"train/track/injector/block_29/weight_norm":71.70845031738281,"train/timing/unit/WanVideoUnit_PromptEmbedder_s":0.00028199722873978317,"train/hbm/reserved_gb":39.55071258544922,"train/track/injector/block_00/grad_norm":0.01542600616812706,"train/track/injector/block_06/weight_norm":71.78461456298828,"train/t5/on_cuda":-1,"train/track/injector/block_04/grad_norm":0.006662918254733086,"train/track/injector/block_18/weight_norm":71.71800231933594,"train/track/injector/block_22/weight_norm":71.71602630615234,"train/track/injector/block_37/grad_norm":0.001781684928573668,"train/timing/other_s":0.6916093489853665,"train/timing/unit/WanVideoUnit_NoiseInitializer_s":0.00017892703181132674,"train/track/injector/block_13/weight_norm":71.73668670654297,"train/track/injector/block_28/weight_norm":71.74348449707031,"train/track/injector/block_07/grad_norm":0.0076438491232693195,"train/track/injector/block_23/weight_norm":71.71672058105469,"train/track/injector/block_15/weight_norm":71.72630310058594,"train/track/injector/block_07/weight_norm":71.78182983398438,"train/track/injector/block_34/grad_norm":0.001947328564710915,"train/track/adapter/weight_norm":17.002622604370117,"train/track/injector/block_10/grad_norm":0.007022695150226355,"train/track/injector/block_09/weight_norm":71.78085327148438,"train/track/injector/block_23/grad_norm":0.003564701182767749,"train/timing/forward_s":6.0979817740008,"train/track/injector/grad_norm_mean":0.004726294428110123,"train/track/injector/block_33/weight_norm":71.6600341796875,"train/timing/microsteps":1,"train/track/injector/block_12/weight_norm":71.75685119628906,"train/track/injector/block_31/grad_norm":0.0020285714417696,"train/track/injector/block_29/grad_norm":0.0028482715133577585,"train/track/injector/block_36/weight_norm":71.64973449707031,"train/track/injector/block_35/weight_norm":71.68326568603516,"train/track/injector/block_10/weight_norm":71.77770233154297,"train/timing/unit/WanVideoUnit_Vista4DStreams_s":5.154164682608098e-05,"train/track/injector/block_14/weight_norm":71.7538833618164,"train/timing/unit/WanVideoUnit_InputVideoEmbedder_XYZCondition_s":1.6001708507537842,"train/track/injector/block_13/grad_norm":0.0054008979350328445,"train/track/injector/block_19/grad_norm":0.00412755599245429,"train/track/injector/block_26/grad_norm":0.0027969893999397755,"train/track/injector/block_02/grad_norm":0.006992134265601635,"train/track/injector/block_21/grad_norm":0.004170014522969723,"train/track/injector/block_38/weight_norm":71.60940551757812,"train/timing/unit/WanVideoUnit_ModeDropoutAugmentation_s":2.9370274205575697e-05,"train/timing/dit_loss_fn_s":2.5807018280029297,"_runtime":518,"train/track/injector/block_05/weight_norm":71.79119110107422,"train/track/injector/weight_norm_max":71.8019027709961,"train/timing/unit/WanVideoUnit_TargetVideoEncoder_s":0.8126809597015381,"train/track/injector/block_04/weight_norm":71.78141784667969,"train/track/injector/block_01/weight_norm":71.73615264892578,"_wandb":{"runtime":518},"train/timing/unit/WanVideoUnit_V2VConcatenate_s":0.00013471435522660613,"train/track/injector/block_06/grad_norm":0.007085887249559164,"train/track/injector/block_28/grad_norm":0.002767539117485285,"train/timing/reduction_s":0.12494410799990874,"train/track/injector/block_39/weight_norm":71.61337280273438,"train/track/injector/block_31/weight_norm":71.70215606689453,"train/timing/unit/WanVideoUnit_ShapeChecker_s":5.647033322020434e-05,"train/timing/unit/WanVideoUnit_CameraControl_NVS_s":0.0032966327853500843,"train/track/injector/block_05/grad_norm":0.007096964865922928,"train/track/injector/block_14/grad_norm":0.005361848510801792,"train/track/injector/block_26/weight_norm":71.73149871826172,"train/track/injector/block_24/grad_norm":0.0032270734664052725,"train/track/injector/block_17/grad_norm":0.0046652089804410934,"_step":2559,"train/track/injector/block_33/grad_norm":0.002168276347219944,"train/track/injector/block_21/weight_norm":71.71273803710938,"train/track/injector/block_11/weight_norm":71.75850677490234,"train/timing/unit/WanVideoUnit_DenseTargetControl_s":0.2419811189174652,"train/track/injector/block_17/weight_norm":71.72594451904297,"train/track/injector/grad_norm_max":0.01542600616812706,"train/track/injector/block_15/grad_norm":0.004815808031708002,"train/timing/unit/WanVideoUnit_TrackPointCondition_s":0.020848633721470833,"train/track/injector/block_19/weight_norm":71.7264633178711,"train/cond_dropout/drop_dense":0.1875,"train/track/injector/block_00/weight_norm":71.62006378173828,"train/epoch":0,"train/track/injector/block_35/grad_norm":0.0018398460233584046,"train/track/injector/block_38/grad_norm":0.002146689221262932,"train/track/injector/block_27/weight_norm":71.7018051147461,"train/track/injector/block_09/grad_norm":0.007202731911092997,"train/track/adapter/grad_norm":0.005933027248829603,"train/track/injector/block_11/grad_norm":0.0059500886127352715,"train/track/injector/block_18/grad_norm":0.004923135042190552,"train/timing/unit/WanVideoUnit_LatentNoiseAugmentation_s":5.086858800495975e-05}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/logs/debug-internal.log

@@ -0,0 +1,87 @@
+{"time":"2026-06-02T06:50:49.974405605Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T06:50:49.976821599Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T06:50:50.36107604Z","level":"INFO","msg":"stream: created new stream","id":"nn58xrpu"}
+{"time":"2026-06-02T06:50:50.36114999Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T06:50:50.362005773Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T06:50:50.362019596Z","level":"INFO","msg":"writer: started","stream_id":"nn58xrpu"}
+{"time":"2026-06-02T06:50:50.362095786Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T06:50:51.690481716Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T06:50:51.888011116Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:01.410733865Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T06:51:01.428514695Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":168}
+{"time":"2026-06-02T06:51:06.691419747Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":122,"uploaded_len":2}
+{"time":"2026-06-02T06:51:06.9238499Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:21.69127157Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T06:51:21.913715771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:36.690922534Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":4,"events_lines":2}
+{"time":"2026-06-02T06:51:36.965752742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:51:51.692183121Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":123,"console_lines":1}
+{"time":"2026-06-02T06:51:51.980180422Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:06.691301178Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":124,"console_lines":2}
+{"time":"2026-06-02T06:52:07.06257193Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:21.69127678Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":2,"events_offset":10,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:52:22.085867072Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:36.69122883Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:52:36.896820176Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:52:51.691346553Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:52:51.892989007Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:06.691797368Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:06.871983169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:21.691461565Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:21.883845999Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:36.693338948Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:36.916559761Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:53:51.69176395Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":22,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:53:51.993609202Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:06.690851286Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":126,"console_lines":1}
+{"time":"2026-06-02T06:54:07.004048439Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:21.691620646Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-06-02T06:54:22.147645092Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:36.690891775Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-06-02T06:54:37.065124365Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:54:51.692454398Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T06:54:51.918197063Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:06.69053338Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T06:55:06.905887225Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:21.691518Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:55:21.947899715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:36.691140706Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:55:36.952030582Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:55:51.691789048Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":2,"events_offset":38,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:55:51.898682683Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:06.691691779Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:06.985943244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:21.696443399Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:21.906715321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:36.691377746Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":2,"events_offset":44,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:36.931509511Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:56:51.691747719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:56:51.960969812Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:06.691390544Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:07.050274115Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:21.69204273Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":50,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:22.186614525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:36.691431427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:36.925377253Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:57:51.691892104Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":54,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:57:51.935668512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:06.691226293Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":56,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:06.90327617Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:21.691228117Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:21.922420057Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:36.691373952Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:36.932965537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:58:51.691882754Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":2,"events_offset":62,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:58:51.913675918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:06.691802803Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:59:06.9232588Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:21.691448365Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":122,"console_lines":1}
+{"time":"2026-06-02T06:59:21.938579666Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:30.473233165Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T06:59:30.474469258Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":127,"console_lines":99,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T06:59:30.683502844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T06:59:30.684680347Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T06:59:30.689378199Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T06:59:30.689457461Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T06:59:30.690169153Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T06:59:30.690201056Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/logs/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 06:50:49,349 INFO    MainThread:199010 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_setup.py:_flush():81] Configure stats pid to 199010
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/logs/debug.log
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/logs/debug-internal.log
+2026-06-02 06:50:49,350 INFO    MainThread:199010 [wandb_init.py:init():848] calling init triggers
+2026-06-02 06:50:49,351 INFO    MainThread:199010 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 0, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 06:50:49,351 INFO    MainThread:199010 [wandb_init.py:init():896] starting backend
+2026-06-02 06:50:49,963 INFO    MainThread:199010 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 06:50:49,972 INFO    MainThread:199010 [wandb_init.py:init():919] backend started and connected
+2026-06-02 06:50:49,975 INFO    MainThread:199010 [wandb_init.py:init():989] updated telemetry
+2026-06-02 06:50:50,021 INFO    MainThread:199010 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 06:50:50,887 INFO    MainThread:199010 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 06:50:51,673 INFO    MainThread:199010 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 06:50:51,674 INFO    MainThread:199010 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 06:50:51,674 INFO    MainThread:199010 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 06:50:51,674 INFO    MainThread:199010 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 06:50:51,682 INFO    MainThread:199010 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 06:59:29,656 INFO    MainThread:199010 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/nn58xrpu
+2026-06-02 06:59:29,663 INFO    MainThread:199010 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 06:59:29,664 INFO    MainThread:199010 [wandb_run.py:_restore():2490] restore
+2026-06-02 06:59:29,664 INFO    MainThread:199010 [wandb_run.py:_restore():2496] restore done
+2026-06-02 06:59:30,688 INFO    MainThread:199010 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_ablate_denseres0_cond_dropout/wandb/run-20260602_065049-nn58xrpu/run-nn58xrpu.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:019ed6c53cc75fb8ede1e41b3a1f83dae28c798b81f4840269386ee945160586
+size 674396

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+#SBATCH --job-name=14b_8node_288x512_pair1x49_next_eb32_lr2e5
+#SBATCH --nodes=8
+#SBATCH --gpus=32
+#SBATCH --gpus-per-node=4
+#SBATCH --ntasks-per-node=1
+#SBATCH --time=24:00:00
+#SBATCH --output=out/exp_%j_%x.out
+
+# Track4D-360 14B FSDP 288x512 next-stage finetune.
+#
+# Main changes from the 288p backendmix OOM-debug launcher:
+#   - new DATA_TAG / output directory;
+#   - warm-start from the latest existing 288p backendmix checkpoint;
+#   - remove most static datasets from DATA_CONFIG;
+#   - add ReCamMaster CoWTracker + square512 source49 + compact pair1x49
+#     VGGT-Omega labels, q1-only by dataset construction.
+#
+# Submit from repo root on Isambard:
+#   sbatch bash_scripts/track4d_360/ismb/14b/sbatch/ismb_sbatch_14b_8node_288x512_fsdp_raymaptokens_nextstage_recammaster_pair1x49_eb32_lr2e5.sh
+
+export NNODES=8
+export MODEL_SIZE=14b
+export CONDA_ENV="${CONDA_ENV:-th29py312}"
+export HEIGHT=288
+export WIDTH=512
+export NUM_FRAMES=49
+export main_process_port="${main_process_port:-29595}"
+
+export WAN_T2V14B_DIR="${WAN_T2V14B_DIR:-${SLURM_SUBMIT_DIR}/weights/Wan2.1-T2V-14B}"
+export VXF_INIT="${VXF_INIT:-${WAN_T2V14B_DIR}/diffusion_pytorch_model*.safetensors}"
+export MODEL_ID_PREFIX="${MODEL_ID_PREFIX:-Wan-AI/Wan2.1-T2V-14B}"
+export TOKENIZER_PATH="${TOKENIZER_PATH:-${WAN_T2V14B_DIR}/google/umt5-xxl}"
+export T5_PATH="${T5_PATH:-${WAN_T2V14B_DIR}/models_t5_umt5-xxl-enc-bf16.pth}"
+export VAE_PATH="${VAE_PATH:-${WAN_T2V14B_DIR}/Wan2.1_VAE.pth}"
+
+# Prefer the current 288p backendmix run as the next-stage warm start. Fall
+# back to the 384p backendmix/raymap-token parents only when no 288p checkpoint exists.
+export BACKENDMIX_288_PARENT_DIR="${BACKENDMIX_288_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_warmstart384_backendmix_q1to4_s1248_jitter_norecam_eb32_288x512_cond_dropout}"
+export BACKENDMIX_384_PARENT_DIR="${BACKENDMIX_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_softsplat_raymaptokens_resume_latest_backendmix_q1to4_s1248_jitter_norecam_eb32_384x672_cond_dropout}"
+export RAYMAP_384_PARENT_DIR="${RAYMAP_384_PARENT_DIR:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout_384x672x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_real_synth_syn4d_recam_syncam_softsplat_raymaptokens_t2v_init_scratch_eb32_cond_dropout}"
+# Warm-start from the 288p backendmix run's step-9000 checkpoint — the requested
+# next-stage starting point. (FSDP1/FSDP2 read it identically: warm-start loads
+# the safetensors BEFORE accelerator.prepare, on the unwrapped model, so it is
+# distributed-backend-independent.) Override WARM_START=... to pick another;
+# falls back to latest backendmix / 384p parents if step-9000 is absent.
+if [ -z "${WARM_START:-}" ]; then
+    if [ -f "${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors" ]; then
+        export WARM_START="${BACKENDMIX_288_PARENT_DIR}/step-9000.safetensors"
+    else
+        latest="$(find "${BACKENDMIX_288_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        if [ -z "${latest}" ]; then
+            latest="$(find "${BACKENDMIX_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        if [ -z "${latest}" ]; then
+            latest="$(find "${RAYMAP_384_PARENT_DIR}" -maxdepth 1 -name 'step-*.safetensors' 2>/dev/null | sort -V | tail -1)"
+        fi
+        export WARM_START="${latest:-${RAYMAP_384_PARENT_DIR}/step-3500.safetensors}"
+    fi
+fi
+
+if ! compgen -G "${VXF_INIT}" >/dev/null; then
+    echo "ERROR: VXF_INIT not found: ${VXF_INIT}" >&2
+    exit 1
+fi
+[ -d "${TOKENIZER_PATH}" ] || { echo "ERROR: TOKENIZER_PATH not found: ${TOKENIZER_PATH}" >&2; exit 1; }
+[ -f "${T5_PATH}" ] || { echo "ERROR: T5_PATH not found: ${T5_PATH}" >&2; exit 1; }
+[ -f "${VAE_PATH}" ] || { echo "ERROR: VAE_PATH not found: ${VAE_PATH}" >&2; exit 1; }
+
+export TARGET_EFFECTIVE_BATCH=32
+export gradient_accumulation_steps=1
+export learning_rate="${learning_rate:-2e-5}"
+export save_steps="${save_steps:-500}"
+export dataset_num_workers="${dataset_num_workers:-7}"
+export TRACK4D360_STOP_AFTER_NEXT_SAVE="${TRACK4D360_STOP_AFTER_NEXT_SAVE:-0}"
+
+export dit_finetune_modules="${dit_finetune_modules:-self_attn}"
+export DIT_FINETUNE_TAG="${DIT_FINETUNE_TAG:-attn}"
+export use_plucker_raymap_tokens=1
+export train_plucker_camera_control=0
+export PLUCKER_WANDB_TAG="${PLUCKER_WANDB_TAG:-plucker_raymap_tokens}"
+
+export DATA_CONFIG="${DATA_CONFIG:-${SLURM_SUBMIT_DIR}/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml}"
+export DATA_TAG="${DATA_TAG:-mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512}"
+
+export RECAMMASTER_SOURCE49_ROOT="${RECAMMASTER_SOURCE49_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}"
+export RECAMMASTER_PAIR1X49_LABEL_ROOT="${RECAMMASTER_PAIR1X49_LABEL_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}"
+export RECAMMASTER_COW_ROOT="${RECAMMASTER_COW_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}"
+export TRACK4D360_RECAM_TRACK_SAMPLE_MODE="${TRACK4D360_RECAM_TRACK_SAMPLE_MODE:-grid}"
+export TRACK4D360_RECAM_TRACK_STRIDE="${TRACK4D360_RECAM_TRACK_STRIDE:-2}"
+export TRACK4D360_RECAM_TRACK_SAMPLES="${TRACK4D360_RECAM_TRACK_SAMPLES:-0}"
+# Pair1x49 labels currently cover only the priority block: first 750 numeric
+# train scenes inside each K/focal group (e.g. train/f18_aperture10/scene000..).
+export TRACK4D360_RECAM_SCENE_OFFSET_PER_K="${TRACK4D360_RECAM_SCENE_OFFSET_PER_K:-0}"
+export TRACK4D360_RECAM_SCENE_LIMIT_PER_K="${TRACK4D360_RECAM_SCENE_LIMIT_PER_K:-750}"
+
+# Fixed post-compaction track identity count — MUST be set (the 24h/10k-step
+# backendmix run had this at 65536; this launcher previously omitted it). The
+# ReCamMaster CoW tracks are dense (grid stride 2, samples=0 = uncapped), so
+# without a cap N varies per sample and a heavy draw spikes the track adapter's
+# proj_down activation (B,N,T,hidden) -> the observed CUDA OOM at
+# conv3d_track_adapter.py proj_down (1.16 GiB alloc, step ~770). Fixing N also
+# pads small draws to a constant shape, eliminating per-step fragmentation. Lower
+# to 32768 for extra headroom on contended nodes.
+export TRACK4D360_TRACK_IDENTITY_FIXED_N="${TRACK4D360_TRACK_IDENTITY_FIXED_N:-65536}"
+
+export TRACK4D360_SPARSE_TRACK_PROB="${TRACK4D360_SPARSE_TRACK_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_STRIDE="${TRACK4D360_SPARSE_TRACK_STRIDE:-2|4|8}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB="${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB:-1.0}"
+export TRACK4D360_SPARSE_TRACK_BLOCK_SIZE="${TRACK4D360_SPARSE_TRACK_BLOCK_SIZE:-64}"
+export TRACK4D360_MULTI_QUERY_PROB="${TRACK4D360_MULTI_QUERY_PROB:-1.0}"
+export TRACK4D360_MULTI_QUERY_CHOICES="${TRACK4D360_MULTI_QUERY_CHOICES:-1|2|3|4}"
+export TRACK4D360_GEOMETRY_NOISE_PROB="${TRACK4D360_GEOMETRY_NOISE_PROB:-1.0}"
+export TRACK4D360_TRACK_OFFSET_STD="${TRACK4D360_TRACK_OFFSET_STD:-0.015}"
+export TRACK4D360_TRACK_TEMPORAL_JITTER_STD="${TRACK4D360_TRACK_TEMPORAL_JITTER_STD:-0.006}"
+export TRACK4D360_TRACK_XY_NOISE_RATIO="${TRACK4D360_TRACK_XY_NOISE_RATIO:-0.25}"
+export TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO="${TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO:-0.5}"
+export TRACK4D360_DEPTH_OFFSET_STD="${TRACK4D360_DEPTH_OFFSET_STD:-0.006}"
+export TRACK4D360_DEPTH_SCALE_JITTER="${TRACK4D360_DEPTH_SCALE_JITTER:-0.01}"
+export TRACK4D360_SOURCE_RGB_NOISE_STD="${TRACK4D360_SOURCE_RGB_NOISE_STD:-0.01}"
+export TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD="${TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD:-0.02}"
+export TRACK4D360_DYNAMIC_MASK_AUG_PROB="${TRACK4D360_DYNAMIC_MASK_AUG_PROB:-0.5}"
+export TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL="${TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL:-5}"
+export TRACK4D360_DYNAMIC_MASK_FLIP_PROB="${TRACK4D360_DYNAMIC_MASK_FLIP_PROB:-0.002}"
+
+export dense_warp_mode="${dense_warp_mode:-hybrid_softsplat}"
+export dense_warp_motion_threshold="${dense_warp_motion_threshold:-0.005}"
+export dense_conditioning_noise_max_level="${dense_conditioning_noise_max_level:-0.15}"
+export source_mask_boundary_depth_dropout_prob="${source_mask_boundary_depth_dropout_prob:-0.5}"
+export source_mask_boundary_kernel_size="${source_mask_boundary_kernel_size:-5}"
+export freeze_track4d_condition_modules="${freeze_track4d_condition_modules:-0}"
+
+export use_gradient_checkpointing=1
+export use_gradient_checkpointing_offload="${use_gradient_checkpointing_offload:-0}"
+export VXF_OFFLOAD_T5_TO_CPU="${VXF_OFFLOAD_T5_TO_CPU:-1}"
+export TRACK4D360_MEM_PROBE="${TRACK4D360_MEM_PROBE:-0}"
+export TRACK4D360_ADAPTER_MEM_PROFILE="${TRACK4D360_ADAPTER_MEM_PROFILE:-0}"
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-${PYTORCH_ALLOC_CONF:-expandable_segments:True}}"
+export PYTORCH_ALLOC_CONF="${PYTORCH_ALLOC_CONF:-${PYTORCH_CUDA_ALLOC_CONF}}"
+export TRACK4D360_DENSE_HYBRID_CACHE_FRAMES="${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES:-7}"
+export TRACK4D360_DENSE_HYBRID_TGT_CHUNK="${TRACK4D360_DENSE_HYBRID_TGT_CHUNK:-2}"
+
+export NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-0}"
+export NCCL_P2P_LEVEL="${NCCL_P2P_LEVEL:-NVL}"
+export NCCL_NVLS_ENABLE=0
+export NCCL_NSOCKS_PERTHREAD=8
+export NCCL_SOCKET_NTHREADS=8
+export NCCL_MAX_NCHANNELS="${NCCL_MAX_NCHANNELS:-32}"
+export NCCL_BUFFSIZE="${NCCL_BUFFSIZE:-8388608}"
+export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
+
+export TRACK4D360_CAPTION_OUTPUT_ROOT="${TRACK4D360_CAPTION_OUTPUT_ROOT:-/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output}"
+export SYN4D_CAPTION_DIR="${SYN4D_CAPTION_DIR:-${TRACK4D360_CAPTION_OUTPUT_ROOT}/Syn4D}"
+
+echo "[raymap-tokens 14B pair1x49 next] T2V init: ${VXF_INIT}"
+echo "[raymap-tokens 14B pair1x49 next] warm start: ${WARM_START}"
+echo "[raymap-tokens 14B pair1x49 next] data: ${DATA_CONFIG}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam source49: ${RECAMMASTER_SOURCE49_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam pair1x49 labels: ${RECAMMASTER_PAIR1X49_LABEL_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam CoW: ${RECAMMASTER_COW_ROOT}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam q1 track sampling: mode=${TRACK4D360_RECAM_TRACK_SAMPLE_MODE}, stride=${TRACK4D360_RECAM_TRACK_STRIDE}, samples=${TRACK4D360_RECAM_TRACK_SAMPLES}"
+echo "[raymap-tokens 14B pair1x49 next] ReCam scene window per K: offset=${TRACK4D360_RECAM_SCENE_OFFSET_PER_K}, limit=${TRACK4D360_RECAM_SCENE_LIMIT_PER_K}"
+echo "[raymap-tokens 14B pair1x49 next] q policy: ReCam q1-only; synthetic choices=${TRACK4D360_MULTI_QUERY_CHOICES}"
+echo "[raymap-tokens 14B pair1x49 next] sparse synthetic tracks: stride_choices=${TRACK4D360_SPARSE_TRACK_STRIDE}, block_jitter_prob=${TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB}"
+echo "[raymap-tokens 14B pair1x49 next] trainables: self_attn + raymap Ecam/camera_projector + active track/dense condition modules"
+echo "[raymap-tokens 14B pair1x49 next] freeze track/dense condition modules: ${freeze_track4d_condition_modules}"
+echo "[raymap-tokens 14B pair1x49 next] batch: effective=32, grad_accum=1, lr=${learning_rate}, save_steps=${save_steps}"
+echo "[raymap-tokens 14B pair1x49 next] memory: grad_ckpt_offload=${use_gradient_checkpointing_offload}, hybrid_cache_frames=${TRACK4D360_DENSE_HYBRID_CACHE_FRAMES}, hybrid_tgt_chunk=${TRACK4D360_DENSE_HYBRID_TGT_CHUNK}, cuda_alloc_conf=${PYTORCH_CUDA_ALLOC_CONF}"
+
+source "${SLURM_SUBMIT_DIR}/bash_scripts/track4d_360/ismb/sbatch_common.sh"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml

@@ -0,0 +1,77 @@
+# Next-stage 288x512 backendmix finetune with cleaned ReCamMaster pair1x49 labels.
+#
+# Mix: 35% ReCamMaster pair1x49, 35% Syn4D, 5% Kubric49, 5% Kubric81,
+# and 20% legacy static datasets. The static 20% keeps the previous backendmix
+# relative weights: DL3DV:RE10K:MVS-Synth:TartanAir = 65000:7500:12500:25000.
+# ReCamMaster uses compact square512 source49 + pair1x49 VGGT-Omega scale
+# labels plus CoWTracker source tracks and is q1-only by construction.
+#
+# sbatch_common.sh still sanity-checks for the old MVS-Synth guard string:
+# exclude_sequence_ids=['0000']. This config has no MVS-Synth entry.
+
+num_views: 49
+resolution: [512, 288]
+
+syn4d_root: ${oc.env:SYN4D_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/Syn4D}
+syn4d_caption_dir: ${oc.env:SYN4D_CAPTION_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/Syn4D}
+syn4d_valid_sequences: configs/syn4d_valid_sequences.txt
+dl3dv_root: ${oc.env:DL3DV_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/dl3dv_tar}
+re10k_root: ${oc.env:RE10K_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_re10k}
+mvs_synth_root: ${oc.env:MVS_SYNTH_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_mvs_synth}
+tartanair_root: ${oc.env:TARTANAIR_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/cut3r_processed/processed_tartanair}
+kubric49_root: ${oc.env:KUBRIC49_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_49frames}
+kubric81_root: ${oc.env:KUBRIC81_ROOT,/lus/lfs1aip2/projects/public/u6cl/yslan_data/track_dataset/train_preprocessed_81frames}
+kubric49_caption_cache_dir: ${oc.env:KUBRIC49_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_49}
+kubric81_caption_cache_dir: ${oc.env:KUBRIC81_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/kubric_train_81}
+recammaster_caption_cache_dir: ${oc.env:RECAMMASTER_CAPTION_CACHE_DIR,/lus/lfs1aip2/projects/public/u6ew/yslan_data/caption_output/recammaster}
+recammaster_root: ${oc.env:RECAMMASTER_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset}
+multicam_fg_mask_root: ${oc.env:MULTICAM_FG_MASK_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_SAM3_FG_81f_sources_person_human_288x512}
+recammaster_source49_root: ${oc.env:RECAMMASTER_SOURCE49_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_square512_source49}
+recammaster_pair1x49_label_root: ${oc.env:RECAMMASTER_PAIR1X49_LABEL_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_4RC_VGGT_Omega_pair1x49_square512_scale_labels}
+recammaster_cow_root: ${oc.env:RECAMMASTER_COW_ROOT,/lus/lfs1aip2/projects/public/u6ew/yslan_public/dataset/MultiCamVideo-Dataset_VGGT_Omega_CoWTracker_SourceTracks_all}
+recammaster_track_sample_mode: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLE_MODE,grid}
+recammaster_track_stride: ${oc.env:TRACK4D360_RECAM_TRACK_STRIDE,2}
+recammaster_track_samples: ${oc.env:TRACK4D360_RECAM_TRACK_SAMPLES,0}
+recammaster_scene_offset_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_OFFSET_PER_K,0}
+recammaster_scene_limit_per_k: ${oc.env:TRACK4D360_RECAM_SCENE_LIMIT_PER_K,750}
+
+sparse_track_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_PROB,1.0}
+sparse_track_stride: ${oc.env:TRACK4D360_SPARSE_TRACK_STRIDE,2|4|8}
+sparse_track_query_stride_mode: ${oc.env:TRACK4D360_SPARSE_TRACK_QUERY_STRIDE_MODE,query_count_floor}
+sparse_track_block_jitter_prob: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_JITTER_PROB,1.0}
+sparse_track_block_size: ${oc.env:TRACK4D360_SPARSE_TRACK_BLOCK_SIZE,64}
+multi_query_prob: ${oc.env:TRACK4D360_MULTI_QUERY_PROB,1.0}
+multi_query_choices: ${oc.env:TRACK4D360_MULTI_QUERY_CHOICES,1|2|3|4}
+geometry_noise_prob: ${oc.env:TRACK4D360_GEOMETRY_NOISE_PROB,1.0}
+track_offset_std: ${oc.env:TRACK4D360_TRACK_OFFSET_STD,0.015}
+track_temporal_jitter_std: ${oc.env:TRACK4D360_TRACK_TEMPORAL_JITTER_STD,0.006}
+track_xy_noise_ratio: ${oc.env:TRACK4D360_TRACK_XY_NOISE_RATIO,0.25}
+track_frame_depth_jitter_ratio: ${oc.env:TRACK4D360_TRACK_FRAME_DEPTH_JITTER_RATIO,0.5}
+depth_offset_std: ${oc.env:TRACK4D360_DEPTH_OFFSET_STD,0.006}
+depth_scale_jitter: ${oc.env:TRACK4D360_DEPTH_SCALE_JITTER,0.01}
+source_rgb_noise_std: ${oc.env:TRACK4D360_SOURCE_RGB_NOISE_STD,0.01}
+source_rgb_frame_jitter_std: ${oc.env:TRACK4D360_SOURCE_RGB_FRAME_JITTER_STD,0.02}
+dynamic_mask_aug_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_AUG_PROB,0.5}
+dynamic_mask_morph_kernel: ${oc.env:TRACK4D360_DYNAMIC_MASK_MORPH_KERNEL,5}
+dynamic_mask_flip_prob: ${oc.env:TRACK4D360_DYNAMIC_MASK_FLIP_PROB,0.002}
+
+train_datasets:
+  # 35% ReCamMaster CoWTracker + square512 source49 + compact pair1x49 VGGT labels.
+  # This dataset is q1 only: it always emits track_query_indices=[0]. The
+  # default track density is grid stride 2 to avoid lifting every CoW pixel
+  # during training; override TRACK4D360_RECAM_TRACK_SAMPLE_MODE=all for dense.
+  - "87500 @ ReCamMasterCowVGGT_NVS(ROOT='${recammaster_root}', vggt_root='${recammaster_source49_root}', source_vggt_root='${recammaster_source49_root}', pair1x49_label_root='${recammaster_pair1x49_label_root}', geometry_source='pair1x49_label_scale_only', cow_root='${recammaster_cow_root}', fg_mask_root='${multicam_fg_mask_root}', caption_cache_dir='${recammaster_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, vggt_post_crop_mode='cow_aspect', rgb_post_crop_mode='cow_aspect', track_sample_mode='${recammaster_track_sample_mode}', track_stride=${recammaster_track_stride}, track_samples=${recammaster_track_samples}, scene_offset_per_k=${recammaster_scene_offset_per_k}, scene_limit_per_k=${recammaster_scene_limit_per_k})"
+  # 35% Syn4D dynamic multi-camera mesh-tracked 3D.
+  - "87500 @ GeometryRobustnessWrapper_NVS(Syn4D_Multi_NVS(ROOT='${syn4d_root}', caption_dir='${syn4d_caption_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=87500, track_spatial_stride=1, exclude_scenes=['flying_group'], valid_sequences_file='${syn4d_valid_sequences}', use_mp4_rgb=True, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 49-frame dynamic data.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric49_root}', caption_cache_dir='${kubric49_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 5% Kubric 81-frame dynamic data sampled to 49.
+  - "12500 @ GeometryRobustnessWrapper_NVS(Kubric_NVS(ROOT='${kubric81_root}', caption_cache_dir='${kubric81_caption_cache_dir}', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=12500, track_spatial_stride=1, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, return_dynamic_masks=True, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}'), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 11.82% DL3DV (real, DA3 depth). Static total is 20%; this keeps the old static relative weight.
+  - "29550 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${dl3dv_root}', dataset_type='dl3dv', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=29550, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=0.5, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, use_da3_depth=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 1.38% RE10K (real, DA3 depth).
+  - "3450 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${re10k_root}', dataset_type='re10k', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=3450, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 2.28% MVS-Synth (synthetic, GT depth; scene 0000 held out).
+  - "5700 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${mvs_synth_root}', dataset_type='mvs_synth', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=5700, track_spatial_stride=1, min_overlap_ratio=0.15, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True, exclude_sequence_ids=['0000']), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"
+  # 4.52% TartanAir (static forward-moving scenes).
+  - "11300 @ GeometryRobustnessWrapper_NVS(StaticScene_NVS(ROOT='${tartanair_root}', dataset_type='tartanair', num_views=${num_views}, resolution=${resolution}, steps_per_epoch=11300, track_spatial_stride=1, min_overlap_ratio=0.15, min_shift_ratio=0.15, max_shift_ratio=0.5, shift_beta_ab=[2.0, 5.0], src_center_in_target_prob=1.0, return_source_depth_video=True, return_source_rgb_video=True, return_source_cameras=True), sparse_track_prob=${sparse_track_prob}, sparse_track_stride='${sparse_track_stride}', sparse_track_query_stride_mode='${sparse_track_query_stride_mode}', sparse_track_block_jitter_prob=${sparse_track_block_jitter_prob}, sparse_track_block_size=${sparse_track_block_size}, multi_query_prob=${multi_query_prob}, multi_query_choices='${multi_query_choices}', noise_prob=${geometry_noise_prob}, track_offset_std=${track_offset_std}, track_temporal_jitter_std=${track_temporal_jitter_std}, track_xy_noise_ratio=${track_xy_noise_ratio}, track_frame_depth_jitter_ratio=${track_frame_depth_jitter_ratio}, depth_offset_std=${depth_offset_std}, depth_scale_jitter=${depth_scale_jitter}, source_rgb_noise_std=${source_rgb_noise_std}, source_rgb_frame_jitter_std=${source_rgb_frame_jitter_std}, dynamic_mask_aug_prob=${dynamic_mask_aug_prob}, dynamic_mask_morph_kernel=${dynamic_mask_morph_kernel}, dynamic_mask_flip_prob=${dynamic_mask_flip_prob})"

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/torch_profiler/rank0.1780391935713307947.pt.trace.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6046891542cb397d3092b7489775e48a8cbd2749d633b85ee960cc5ee89dab43
+size 4806779109

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/debug-internal.log

@@ -0,0 +1,89 @@
+{"time":"2026-06-02T09:15:03.360626307Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-02T09:15:03.363384731Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
+{"time":"2026-06-02T09:15:03.763480731Z","level":"INFO","msg":"stream: created new stream","id":"iwxrpu9h"}
+{"time":"2026-06-02T09:15:03.76358316Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-02T09:15:03.764307429Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-02T09:15:03.764319332Z","level":"INFO","msg":"writer: started","stream_id":"iwxrpu9h"}
+{"time":"2026-06-02T09:15:03.764409442Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-02T09:15:05.306162953Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-06-02T09:15:05.54706583Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:15:12.213355249Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":44}
+{"time":"2026-06-02T09:15:12.218119827Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":187}
+{"time":"2026-06-02T09:15:20.30658052Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":133,"uploaded_len":2}
+{"time":"2026-06-02T09:15:20.50100822Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:15:35.307091233Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
+{"time":"2026-06-02T09:15:35.619078665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:15:50.306475896Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":134,"console_lines":4}
+{"time":"2026-06-02T09:15:50.487338735Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:16:05.306770422Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":6,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:16:05.521198844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:16:20.306562945Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":8,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:16:20.500086028Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:16:35.306929037Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":2,"events_offset":10,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:16:35.544966706Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:16:50.306544421Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:16:50.521662086Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:17:05.307011463Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:17:05.513840193Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:17:20.306873464Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:17:20.524164031Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:17:35.306466717Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:17:35.513725414Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:17:50.306691479Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:17:50.509930633Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:18:05.306781235Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
+{"time":"2026-06-02T09:18:05.487008517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:18:20.306238049Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
+{"time":"2026-06-02T09:18:20.594952487Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:18:35.307123907Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
+{"time":"2026-06-02T09:18:35.519206589Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:18:50.307778255Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
+{"time":"2026-06-02T09:18:50.697208883Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:19:05.306718696Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
+{"time":"2026-06-02T09:19:05.753941277Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:19:20.306716421Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
+{"time":"2026-06-02T09:19:20.616359066Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:19:35.307089272Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:19:35.825088137Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:19:50.306659894Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:19:50.705064787Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:20:05.306909037Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":38,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:20:06.265337713Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:20:20.307661911Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":2,"events_offset":40,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:20:20.531282276Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:20:35.306992262Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:20:35.564305086Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:20:50.307023536Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:20:50.635389586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:21:05.307044407Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:21:05.583228743Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:21:20.307285592Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:21:20.508409608Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:21:35.306917129Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:21:35.509511474Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:21:50.307082921Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:21:50.495889909Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:22:05.307152941Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":54,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:22:05.500531686Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:22:20.307303758Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:22:20.526745051Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:22:35.307493135Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:22:35.506899762Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:22:50.306876249Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":2,"events_offset":60,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:22:50.50747968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:23:05.307030151Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:23:05.506590001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:23:20.306698019Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:23:20.500838005Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:23:35.306631161Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":2,"events_offset":66,"events_lines":2,"console_offset":133,"console_lines":1}
+{"time":"2026-06-02T09:23:35.526000016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:23:50.30648744Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":138,"console_lines":110}
+{"time":"2026-06-02T09:23:50.517760083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:23:51.771796788Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-06-02T09:23:51.772448483Z","level":"INFO","msg":"filestream: sending request","total_files":1,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-06-02T09:23:52.050571134Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-06-02T09:23:52.056627937Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-06-02T09:23:52.063523854Z","level":"INFO","msg":"stream: closing"}
+{"time":"2026-06-02T09:23:52.063589901Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-06-02T09:23:52.06482974Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-06-02T09:23:52.064859276Z","level":"INFO","msg":"stream: closed"}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-06-02 09:15:02,858 INFO    MainThread:236392 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
+2026-06-02 09:15:02,858 INFO    MainThread:236392 [wandb_setup.py:_flush():81] Configure stats pid to 236392
+2026-06-02 09:15:02,858 INFO    MainThread:236392 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-02 09:15:02,858 INFO    MainThread:236392 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/logs/debug.log
+2026-06-02 09:15:02,859 INFO    MainThread:236392 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/logs/debug-internal.log
+2026-06-02 09:15:02,859 INFO    MainThread:236392 [wandb_init.py:init():848] calling init triggers
+2026-06-02 09:15:02,859 INFO    MainThread:236392 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
+config: {'learning_rate': 2e-05, 'weight_decay': 0.01, 'num_epochs': 200, 'save_steps': 500, 'num_workers': 7, 'prefetch_factor': 2, 'dataset_base_path': None, 'dataset_metadata_path': None, 'dataset_repeat': 1, 'dataset_num_workers': 7, 'dataloader_prefetch_factor': 2, 'data_file_keys': 'image,video', 'pointmap_stats_path': None, 'model_paths': None, 'model_id_with_origin_paths': 'Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth', 'model_variant': 'vt2v', 'extra_inputs': None, 'track_xyz_normalization': 'minmax', 'track_xyz_quantile_low': 0.02, 'track_xyz_quantile_high': 0.98, 'predict_track_residual': False, 'residual_target_xyz_source': 'clean', 'residual_zero_center_normalization': False, 'residual_linear_normalization': False, 'residual_log_scale_factor': 3.0, 'stream3r_common_scale_stat': 'mean', 'explicit_motion_loss_weight': 0.0, 'explicit_motion_loss_snr_weight': True, 'explicit_motion_loss_snr_sigmoid': True, 'explicit_motion_loss_snr_sigmoid_a': 1.0, 'explicit_motion_loss_snr_sigmoid_b': -5.0, 'explicit_motion_loss_decode_checkpoint': False, 'explicit_motion_loss_decode_chunk_size': 0, 'normalize_track_with_xyz_cond_stats': False, 'prenormalized_track_video_signed_log': False, 'use_xyz_video_condition': False, 'xyz_video_condition_scale': 1.0, 'xyz_video_condition_aug_prob': 0.0, 'xyz_video_condition_noise_std': 0.0, 'xyz_video_condition_z_noise_std': 0.0, 'xyz_video_condition_frame_scale_std': 0.0, 'xyz_video_condition_dropout_prob': 0.0, 'xyz_video_condition_mode': 'xyz', 'xyz_video_condition_cfg_dropout_prob': 0.0, 'camera_condition_cfg_dropout_prob': 0.0, 'fp8_models': None, 'offload_models': None, 'v2v_concat_mode': 'width', 'v2v_channel_patch_embed_mode': 'add', 'trainable_models': 'dit.domain_embedding,dit.control_patch_embedding', 'find_unused_parameters': False, 'task': 'sft:v2v', 'drop_text_prob': 0.1, 'drop_video_prob': 0.0, 'drop_t2v_prob': None, 'drop_i2v_prob': None, 'input_video_noise_strength': 0.0, 'input_video_noise_random_max': 500, 'output_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout', 'remove_prefix_in_ckpt': 'pipe.dit.', 'save_every_n_epochs': 2, 'max_checkpoints_to_keep': 5, 'allow_existing_checkpoints': True, 'initial_step': 2530, 'max_train_steps': 2560, 'lora_base_model': None, 'lora_target_modules': 'q,k,v,o,ffn.0,ffn.2', 'lora_rank': 32, 'lora_alpha': None, 'lora_checkpoint': None, 'preset_lora_path': None, 'preset_lora_model': None, 'use_gradient_checkpointing': True, 'use_gradient_checkpointing_offload': False, 'gradient_accumulation_steps': 1, 'use_wandb': True, 'wandb_project': 'track4d-360', 'wandb_name': 'ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp', 'wandb_entity': None, 'wandb_tags': ['ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug'], 'wandb_notes': None, 'wandb_log_gradients': False, 'wandb_log_parameters': False, 'wandb_log_freq': 100, 'wandb_watch_model': False, 'debug_latents': False, 'debug_latents_every': 500, 'debug_latents_fps': 16, 'debug_latents_quality': 5, 'debug_latents_max_frames': None, 'debug_latents_outdir': 'debug_latents', 'debug_latents_vae_tiled': True, 'debug_latents_vae_tile_size': (30, 52), 'debug_latents_vae_tile_stride': (15, 26), 'debug_missing_grads': False, 'torch_distributed_debug': None, 'use_query_frame_impulse_condition': True, 'height': 288, 'width': 512, 'max_pixels': 1048576, 'num_frames': 49, 'tokenizer_path': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl', 'dit_finetune_modules': 'self_attn', 'track_injection_mode': 'per_block', 'track_injection_block_mode': 'concat_project', 'train_sigma_shift': 5.0, 'dummy': False, 'steps_per_epoch': 25000, 'initialize_model_on_cpu': False, 'resume_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors', 'gs_head_init_checkpoint': None, 'joint_track_prediction': False, 'joint_track_depth_prediction': False, 'joint_depth_prediction': False, 'target_depth_log_cap': 14.0, 'target_depth_norm_type': 'log_depth', 'target_depth_max_disparity': 5.0, 'vxf_init_checkpoint': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors', 'channel_concat_stage': None, 'use_residual_tracks': True, 'freeze_lora_after_load': False, 'skip_v2v_required_trainables': False, 'gs_only_training': False, 'freeze_track_block_injector': False, 'freeze_track4d_condition_modules': False, 'gs_head_style': 'flashworld', 'use_gs_output': False, 'gs_zero_features': False, 'gs_feat_dim': 1024, 'gs_loss_weight': 1.0, 'gs_mse_loss_weight': 1.0, 'gs_lpips_loss_weight': 0.5, 'gs_depth_loss_weight': 0.05, 'gs_abs_depth_loss_weight': 0.0, 'gs_opacity_loss_weight': 0.1, 'gs_opacity_prune_keep_frac': 1.0, 'gs_rgb_loss_weight': None, 'gs_sh_degree': 2, 'gs_num_points_per_pixel': 2, 'no_gs_latent_detach': False, 'use_gs_feedback_training': False, 'gs_scale_cap': None, 'gs_render_patch_size': None, 'gs_legacy_union_render': False, 'gs_lyra_depth_param': 'exp', 'gs_lyra_log_depth_shift': 0.0, 'gs_lyra_dnear': 0.1, 'gs_lyra_dfar': 10.0, 'gs_replay_cache_size': 0, 'gs_dynamic_mode': 'static', 'gs_track_supervision_mode': 'pointmap', 'gs_pos_loss_weight': 1.0, 'gs_dyn_depth_loss_weight': 0.5, 'gs_opa_tv_weight': 0.01, 'gs_xyz_tv_weight': 0.01, 'gs_predict_rotation': False, 'gs_disable_delta_opacity': False, 'gs_direct_pos_loss_weight': 0.0, 'gs_track_canonical_loss_weight': 0.0, 'gs_track_delta_loss_weight': 0.0, 'gs_track_final_pos_loss_weight': 0.0, 'gs_dynamic_max_t_video': 64, 'gs_assoc_track_2d_loss_weight': 0.0, 'gs_track_accel_loss_weight': 0.0, 'gs_track_loss_quantile': 0.98, 'gs_track_canonical_detach': False, 'gs_track_joint_render': False, 'cond_drop_track_prob': 0.1, 'cond_drop_dense_prob': 0.0, 'cond_drop_geo_prob': 0.1, 'use_track_null_token_fallback': False, 'use_dense_null_token_fallback': False, 'use_plucker_camera_control': True, 'enable_v2v_plucker_camera_control': True, 'train_plucker_camera_control': False, 'use_plucker_raymap_tokens': True, 'plucker_use_ray_origin': False, 'use_perlayer_camera_control': True, 'camera_injection_mode': 'projector', 'camera_plucker_encoder_mode': 'patchify', 'warped_condition_mode': 'off', 'warped_appearance_fusion': 'concat_proj', 'warped_geom_only_dense': True, 'multi_frame_warped_appearance': False, 'multi_frame_motion_threshold': 0.01, 'fixed_step_frac': None, 'fixed_num_inference_steps': 50, 'fixed_sigma': None, 'skip_diffusion_loss': False, 'use_dense_branch': True, 'dense_in_channels': 5, 'dense_proj_dim': 32, 'dense_num_residual_blocks': 2, 'dense_alpha_track': 1.0, 'use_depth_null_token': False, 'dense_conditioning_noise_max_level': 0.15, 'source_mask_boundary_depth_dropout_prob': 0.5, 'source_mask_boundary_kernel_size': 5, 'dense_warp_mode': 'hybrid_softsplat', 'dense_warp_motion_threshold': 0.005, 'dense_hybrid_disable_dynamic_masks': False, 'use_vista4d_latent_streams': False, 'use_gen3c_warp_latent_channels': False, 'disable_track_modules': False, 'dense_only_condition_modules': False, 'torch_profile_active_steps': 5, 'torch_profile_wait_steps': 3, 'torch_profile_warmup_steps': 1, 'torch_profile_repeat': 1, 'torch_profile_dir': None, 'torch_profile_record_shapes': False, 'torch_profile_memory': False, 'torch_profile_with_stack': False, 'track_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml', 'track4d360_data_config': '/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml', 'syn4d_root': '/scratch/shared/beegfs/zeren/Syn4D', 'scene': None, 'sequence_idx': None, 'track_spatial_stride': 1, '_wandb': {}}
+2026-06-02 09:15:02,859 INFO    MainThread:236392 [wandb_init.py:init():896] starting backend
+2026-06-02 09:15:03,349 INFO    MainThread:236392 [wandb_init.py:init():911] sending inform_init request
+2026-06-02 09:15:03,357 INFO    MainThread:236392 [wandb_init.py:init():919] backend started and connected
+2026-06-02 09:15:03,360 INFO    MainThread:236392 [wandb_init.py:init():989] updated telemetry
+2026-06-02 09:15:03,426 INFO    MainThread:236392 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
+2026-06-02 09:15:04,130 INFO    MainThread:236392 [wandb_init.py:init():1058] starting run threads in backend
+2026-06-02 09:15:05,292 INFO    MainThread:236392 [wandb_run.py:_console_start():2542] atexit reg
+2026-06-02 09:15:05,292 INFO    MainThread:236392 [wandb_run.py:_redirect():2391] redirect: wrap_raw
+2026-06-02 09:15:05,293 INFO    MainThread:236392 [wandb_run.py:_redirect():2460] Wrapping output streams.
+2026-06-02 09:15:05,293 INFO    MainThread:236392 [wandb_run.py:_redirect():2483] Redirects installed.
+2026-06-02 09:15:05,299 INFO    MainThread:236392 [wandb_init.py:init():1098] run started, returning control to user process
+2026-06-02 09:23:51,064 INFO    MainThread:236392 [wandb_run.py:_finish():2309] finishing run nirvanalan/track4d-360/iwxrpu9h
+2026-06-02 09:23:51,067 INFO    MainThread:236392 [wandb_run.py:_atexit_cleanup():2508] got exitcode: 0
+2026-06-02 09:23:51,068 INFO    MainThread:236392 [wandb_run.py:_restore():2490] restore
+2026-06-02 09:23:51,069 INFO    MainThread:236392 [wandb_run.py:_restore():2496] restore done
+2026-06-02 09:23:52,062 INFO    MainThread:236392 [wandb_run.py:_footer_sync_info():3886] logging synced files

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/files/config.yaml

@@ -0,0 +1,700 @@
+_wandb:
+    value:
+        cli_version: 0.26.0
+        e:
+            f0ztrk8t2qi0av8h7ptk8q57xoxkau1o:
+                args:
+                    - --model_id_with_origin_paths
+                    - Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+                    - --tokenizer_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+                    - --task
+                    - sft:v2v
+                    - --model_variant
+                    - vt2v
+                    - --v2v_concat_mode
+                    - width
+                    - --height
+                    - "288"
+                    - --width
+                    - "512"
+                    - --num_frames
+                    - "49"
+                    - --track4d360_data_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+                    - --steps_per_epoch
+                    - "25000"
+                    - --num_epochs
+                    - "200"
+                    - --learning_rate
+                    - "2e-5"
+                    - --gradient_accumulation_steps
+                    - "1"
+                    - --save_steps
+                    - "500"
+                    - --remove_prefix_in_ckpt
+                    - pipe.dit.
+                    - --trainable_models
+                    - dit.domain_embedding,dit.control_patch_embedding
+                    - --dit_finetune_modules
+                    - self_attn
+                    - --track_injection_mode
+                    - per_block
+                    - --track_injection_block_mode
+                    - concat_project
+                    - --track_config
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml
+                    - --train_sigma_shift
+                    - "5.0"
+                    - --use_query_frame_impulse_condition
+                    - --use_plucker_camera_control
+                    - --enable_v2v_plucker_camera_control
+                    - --use_plucker_raymap_tokens
+                    - --use_perlayer_camera_control
+                    - --camera_injection_mode
+                    - projector
+                    - --camera_plucker_encoder_mode
+                    - patchify
+                    - --use_dense_branch
+                    - --dense_in_channels
+                    - "5"
+                    - --dense_proj_dim
+                    - "32"
+                    - --dense_num_residual_blocks
+                    - "2"
+                    - --dense_alpha_track
+                    - "1.0"
+                    - --dense_conditioning_noise_max_level
+                    - "0.15"
+                    - --source_mask_boundary_depth_dropout_prob
+                    - "0.5"
+                    - --source_mask_boundary_kernel_size
+                    - "5"
+                    - --dense_warp_mode
+                    - hybrid_softsplat
+                    - --dense_warp_motion_threshold
+                    - "0.005"
+                    - --cond_drop_geo_prob
+                    - "0.1"
+                    - --cond_drop_track_prob
+                    - "0.1"
+                    - --cond_drop_dense_prob
+                    - "0.0"
+                    - --initial_step
+                    - "2530"
+                    - --output_path
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout
+                    - --dataset_num_workers
+                    - "7"
+                    - --dataloader_prefetch_factor
+                    - "2"
+                    - --drop_text_prob
+                    - "0.1"
+                    - --debug_latents_every
+                    - "500"
+                    - --allow_existing_checkpoints
+                    - --wandb_project
+                    - track4d-360
+                    - --wandb_name
+                    - ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp
+                    - --wandb_tags
+                    - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+                    - --use_wandb
+                    - --vxf_init_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+                    - --use_gradient_checkpointing
+                    - --resume_checkpoint
+                    - /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+                    - --max_train_steps
+                    - "2560"
+                    - --torch_profile_active_steps
+                    - "5"
+                    - --torch_profile_wait_steps
+                    - "3"
+                    - --torch_profile_warmup_steps
+                    - "1"
+                    - --torch_profile_repeat
+                    - "1"
+                    - --no_torch_profile_record_shapes
+                    - --no_torch_profile_memory
+                codePath: scripts/wan2.1_fun/train_track4d360.py
+                codePathLocal: scripts/wan2.1_fun/train_track4d360.py
+                cpu_count: 288
+                cpu_count_logical: 288
+                cudaVersion: "12.7"
+                disk:
+                    /:
+                        total: "255795724288"
+                        used: "549191680"
+                email: yushi.lan@outlook.com
+                executable: /projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12
+                git:
+                    commit: e47380b0c0f4cbb87efcbad540b96a404c2ccc88
+                    remote: git@github.com:NIRVANALAN/track4d-360.git
+                gpu: NVIDIA GH200 120GB
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-d655252a-055e-6123-1fcc-96c384e94cb6
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-f32c462b-35e6-cd39-3ab1-ef2d12935745
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-c6448e24-bbf3-db8b-e6c7-7becaf680d11
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "102625181696"
+                      name: NVIDIA GH200 120GB
+                      uuid: GPU-2c148b70-ecf2-fb15-969b-cbb11a8b9bc4
+                host: nid011121
+                memory:
+                    total: "919613407232"
+                os: Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38
+                program: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py
+                python: CPython 3.12.13
+                root: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout
+                slurm:
+                    cluster_name: gracehopper
+                    conf: /var/spool/slurmd/conf-cache/slurm.conf
+                    cpu_bind: quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_list: 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+                    cpu_bind_type: 'mask_cpu:'
+                    cpu_bind_verbose: quiet
+                    cpus_on_node: "288"
+                    distribution: cyclic
+                    gpus: "32"
+                    gpus_on_node: "4"
+                    gpus_per_node: "4"
+                    gtids: "0"
+                    job_account: brics.u6ew
+                    job_cpus_per_node: 288(x8)
+                    job_end_time: "1780398532"
+                    job_gid: "1483804847"
+                    job_gpus: 0,1,2,3
+                    job_id: "4984787"
+                    job_name: 14b_pair1x49_profile_patchifylinear_fsdp2
+                    job_nodelist: nid[011121-011122,011125,011127-011129,011165,011167]
+                    job_num_nodes: "8"
+                    job_partition: workq
+                    job_qos: normal
+                    job_start_time: "1780391332"
+                    job_uid: "1483804847"
+                    job_user: yushi.u6ew
+                    jobid: "4984787"
+                    launch_node_ipaddr: 10.242.4.57
+                    localid: "0"
+                    mpi_type: pmi2
+                    nnodes: "8"
+                    nodeid: "0"
+                    nodelist: nid[011121-011122,011125,011127-011129,011165,011167]
+                    nprocs: "8"
+                    ntasks: "8"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    srun_comm_host: 10.242.4.57
+                    srun_comm_port: "45551"
+                    step_gpus: 0,1,2,3
+                    step_id: "0"
+                    step_launcher_port: "45551"
+                    step_nodelist: nid[011121-011122,011125,011127-011129,011165,011167]
+                    step_num_nodes: "8"
+                    step_num_tasks: "8"
+                    step_resv_ports: 27631-27632
+                    step_tasks_per_node: 1(x8)
+                    stepid: "0"
+                    submit_dir: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360
+                    submit_host: login40
+                    task_pid: "236308"
+                    tasks_per_node: 1(x8)
+                    topology_addr: global.group12.nid011121
+                    topology_addr_pattern: switch.switch.node
+                    umask: "0022"
+                startedAt: "2026-06-02T09:15:02.848946Z"
+                writerId: f0ztrk8t2qi0av8h7ptk8q57xoxkau1o
+        m: []
+        python_version: 3.12.13
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 98
+                - 106
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 83
+                - 98
+                - 106
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 41
+                - 61
+            "4": 3.12.13
+            "5": 0.26.0
+            "6": 4.46.2
+            "12": 0.26.0
+            "13": linux-aarch64
+allow_existing_checkpoints:
+    value: true
+camera_condition_cfg_dropout_prob:
+    value: 0
+camera_injection_mode:
+    value: projector
+camera_plucker_encoder_mode:
+    value: patchify
+channel_concat_stage:
+    value: null
+cond_drop_dense_prob:
+    value: 0
+cond_drop_geo_prob:
+    value: 0.1
+cond_drop_track_prob:
+    value: 0.1
+data_file_keys:
+    value: image,video
+dataloader_prefetch_factor:
+    value: 2
+dataset_base_path:
+    value: null
+dataset_metadata_path:
+    value: null
+dataset_num_workers:
+    value: 7
+dataset_repeat:
+    value: 1
+debug_latents:
+    value: false
+debug_latents_every:
+    value: 500
+debug_latents_fps:
+    value: 16
+debug_latents_max_frames:
+    value: null
+debug_latents_outdir:
+    value: debug_latents
+debug_latents_quality:
+    value: 5
+debug_latents_vae_tile_size:
+    value:
+        - 30
+        - 52
+debug_latents_vae_tile_stride:
+    value:
+        - 15
+        - 26
+debug_latents_vae_tiled:
+    value: true
+debug_missing_grads:
+    value: false
+dense_alpha_track:
+    value: 1
+dense_conditioning_noise_max_level:
+    value: 0.15
+dense_hybrid_disable_dynamic_masks:
+    value: false
+dense_in_channels:
+    value: 5
+dense_num_residual_blocks:
+    value: 2
+dense_only_condition_modules:
+    value: false
+dense_proj_dim:
+    value: 32
+dense_warp_mode:
+    value: hybrid_softsplat
+dense_warp_motion_threshold:
+    value: 0.005
+disable_track_modules:
+    value: false
+dit_finetune_modules:
+    value: self_attn
+drop_i2v_prob:
+    value: null
+drop_t2v_prob:
+    value: null
+drop_text_prob:
+    value: 0.1
+drop_video_prob:
+    value: 0
+dummy:
+    value: false
+enable_v2v_plucker_camera_control:
+    value: true
+explicit_motion_loss_decode_checkpoint:
+    value: false
+explicit_motion_loss_decode_chunk_size:
+    value: 0
+explicit_motion_loss_snr_sigmoid:
+    value: true
+explicit_motion_loss_snr_sigmoid_a:
+    value: 1
+explicit_motion_loss_snr_sigmoid_b:
+    value: -5
+explicit_motion_loss_snr_weight:
+    value: true
+explicit_motion_loss_weight:
+    value: 0
+extra_inputs:
+    value: null
+find_unused_parameters:
+    value: false
+fixed_num_inference_steps:
+    value: 50
+fixed_sigma:
+    value: null
+fixed_step_frac:
+    value: null
+fp8_models:
+    value: null
+freeze_lora_after_load:
+    value: false
+freeze_track_block_injector:
+    value: false
+freeze_track4d_condition_modules:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gs_abs_depth_loss_weight:
+    value: 0
+gs_assoc_track_2d_loss_weight:
+    value: 0
+gs_depth_loss_weight:
+    value: 0.05
+gs_direct_pos_loss_weight:
+    value: 0
+gs_disable_delta_opacity:
+    value: false
+gs_dyn_depth_loss_weight:
+    value: 0.5
+gs_dynamic_max_t_video:
+    value: 64
+gs_dynamic_mode:
+    value: static
+gs_feat_dim:
+    value: 1024
+gs_head_init_checkpoint:
+    value: null
+gs_head_style:
+    value: flashworld
+gs_legacy_union_render:
+    value: false
+gs_loss_weight:
+    value: 1
+gs_lpips_loss_weight:
+    value: 0.5
+gs_lyra_depth_param:
+    value: exp
+gs_lyra_dfar:
+    value: 10
+gs_lyra_dnear:
+    value: 0.1
+gs_lyra_log_depth_shift:
+    value: 0
+gs_mse_loss_weight:
+    value: 1
+gs_num_points_per_pixel:
+    value: 2
+gs_only_training:
+    value: false
+gs_opa_tv_weight:
+    value: 0.01
+gs_opacity_loss_weight:
+    value: 0.1
+gs_opacity_prune_keep_frac:
+    value: 1
+gs_pos_loss_weight:
+    value: 1
+gs_predict_rotation:
+    value: false
+gs_render_patch_size:
+    value: null
+gs_replay_cache_size:
+    value: 0
+gs_rgb_loss_weight:
+    value: null
+gs_scale_cap:
+    value: null
+gs_sh_degree:
+    value: 2
+gs_track_accel_loss_weight:
+    value: 0
+gs_track_canonical_detach:
+    value: false
+gs_track_canonical_loss_weight:
+    value: 0
+gs_track_delta_loss_weight:
+    value: 0
+gs_track_final_pos_loss_weight:
+    value: 0
+gs_track_joint_render:
+    value: false
+gs_track_loss_quantile:
+    value: 0.98
+gs_track_supervision_mode:
+    value: pointmap
+gs_xyz_tv_weight:
+    value: 0.01
+gs_zero_features:
+    value: false
+height:
+    value: 288
+initial_step:
+    value: 2530
+initialize_model_on_cpu:
+    value: false
+input_video_noise_random_max:
+    value: 500
+input_video_noise_strength:
+    value: 0
+joint_depth_prediction:
+    value: false
+joint_track_depth_prediction:
+    value: false
+joint_track_prediction:
+    value: false
+learning_rate:
+    value: 2e-05
+lora_alpha:
+    value: null
+lora_base_model:
+    value: null
+lora_checkpoint:
+    value: null
+lora_rank:
+    value: 32
+lora_target_modules:
+    value: q,k,v,o,ffn.0,ffn.2
+max_checkpoints_to_keep:
+    value: 5
+max_pixels:
+    value: 1048576
+max_train_steps:
+    value: 2560
+model_id_with_origin_paths:
+    value: Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth
+model_paths:
+    value: null
+model_variant:
+    value: vt2v
+multi_frame_motion_threshold:
+    value: 0.01
+multi_frame_warped_appearance:
+    value: false
+no_gs_latent_detach:
+    value: false
+normalize_track_with_xyz_cond_stats:
+    value: false
+num_epochs:
+    value: 200
+num_frames:
+    value: 49
+num_workers:
+    value: 7
+offload_models:
+    value: null
+output_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout
+plucker_use_ray_origin:
+    value: false
+pointmap_stats_path:
+    value: null
+predict_track_residual:
+    value: false
+prefetch_factor:
+    value: 2
+prenormalized_track_video_signed_log:
+    value: false
+preset_lora_model:
+    value: null
+preset_lora_path:
+    value: null
+remove_prefix_in_ckpt:
+    value: pipe.dit.
+residual_linear_normalization:
+    value: false
+residual_log_scale_factor:
+    value: 3
+residual_target_xyz_source:
+    value: clean
+residual_zero_center_normalization:
+    value: false
+resume_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors
+save_every_n_epochs:
+    value: 2
+save_steps:
+    value: 500
+scene:
+    value: null
+sequence_idx:
+    value: null
+skip_diffusion_loss:
+    value: false
+skip_v2v_required_trainables:
+    value: false
+source_mask_boundary_depth_dropout_prob:
+    value: 0.5
+source_mask_boundary_kernel_size:
+    value: 5
+steps_per_epoch:
+    value: 25000
+stream3r_common_scale_stat:
+    value: mean
+syn4d_root:
+    value: /scratch/shared/beegfs/zeren/Syn4D
+target_depth_log_cap:
+    value: 14
+target_depth_max_disparity:
+    value: 5
+target_depth_norm_type:
+    value: log_depth
+task:
+    value: sft:v2v
+tokenizer_path:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl
+torch_distributed_debug:
+    value: null
+torch_profile_active_steps:
+    value: 5
+torch_profile_dir:
+    value: null
+torch_profile_memory:
+    value: false
+torch_profile_record_shapes:
+    value: false
+torch_profile_repeat:
+    value: 1
+torch_profile_wait_steps:
+    value: 3
+torch_profile_warmup_steps:
+    value: 1
+torch_profile_with_stack:
+    value: false
+track_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml
+track_injection_block_mode:
+    value: concat_project
+track_injection_mode:
+    value: per_block
+track_spatial_stride:
+    value: 1
+track_xyz_normalization:
+    value: minmax
+track_xyz_quantile_high:
+    value: 0.98
+track_xyz_quantile_low:
+    value: 0.02
+track4d360_data_config:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml
+train_plucker_camera_control:
+    value: false
+train_sigma_shift:
+    value: 5
+trainable_models:
+    value: dit.domain_embedding,dit.control_patch_embedding
+use_dense_branch:
+    value: true
+use_dense_null_token_fallback:
+    value: false
+use_depth_null_token:
+    value: false
+use_gen3c_warp_latent_channels:
+    value: false
+use_gradient_checkpointing:
+    value: true
+use_gradient_checkpointing_offload:
+    value: false
+use_gs_feedback_training:
+    value: false
+use_gs_output:
+    value: false
+use_perlayer_camera_control:
+    value: true
+use_plucker_camera_control:
+    value: true
+use_plucker_raymap_tokens:
+    value: true
+use_query_frame_impulse_condition:
+    value: true
+use_residual_tracks:
+    value: true
+use_track_null_token_fallback:
+    value: false
+use_vista4d_latent_streams:
+    value: false
+use_wandb:
+    value: true
+use_xyz_video_condition:
+    value: false
+v2v_channel_patch_embed_mode:
+    value: add
+v2v_concat_mode:
+    value: width
+vxf_init_checkpoint:
+    value: /lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors
+wandb_entity:
+    value: null
+wandb_log_freq:
+    value: 100
+wandb_log_gradients:
+    value: false
+wandb_log_parameters:
+    value: false
+wandb_name:
+    value: ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp
+wandb_notes:
+    value: null
+wandb_project:
+    value: track4d-360
+wandb_tags:
+    value:
+        - ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug
+wandb_watch_model:
+    value: false
+warped_appearance_fusion:
+    value: concat_proj
+warped_condition_mode:
+    value: "off"
+warped_geom_only_dense:
+    value: true
+weight_decay:
+    value: 0.01
+width:
+    value: 512
+xyz_video_condition_aug_prob:
+    value: 0
+xyz_video_condition_cfg_dropout_prob:
+    value: 0
+xyz_video_condition_dropout_prob:
+    value: 0
+xyz_video_condition_frame_scale_std:
+    value: 0
+xyz_video_condition_mode:
+    value: xyz
+xyz_video_condition_noise_std:
+    value: 0
+xyz_video_condition_scale:
+    value: 1
+xyz_video_condition_z_noise_std:
+    value: 0

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/files/output.log

@@ -0,0 +1,248 @@
+✓ WandB initialized: https://wandb.ai/nirvanalan/track4d-360/runs/iwxrpu9h
+/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/lib/python3.12/site-packages/accelerate/utils/fsdp_utils.py:742: UserWarning: FSDP upcast of low precision parameters to fp32 (since mixed_precision != 'no') may affect the precision of model checkpoints. This effects 597 parameters: ['pipe.dit.domain_embedding', 'pipe.dit.blocks.0.self_attn.q.weight', 'pipe.dit.blocks.0.self_attn.q.bias', 'pipe.dit.blocks.0.self_attn.k.weight', 'pipe.dit.blocks.0.self_attn.k.bias', 'pipe.dit.blocks.0.self_attn.v.weight', 'pipe.dit.blocks.0.self_attn.v.bias', 'pipe.dit.blocks.0.self_attn.o.weight', 'pipe.dit.blocks.0.self_attn.o.bias', 'pipe.dit.blocks.0.self_attn.norm_q.weight', 'pipe.dit.blocks.0.self_attn.norm_k.weight', 'pipe.dit.blocks.0.camera_projector.weight', 'pipe.dit.blocks.0.camera_projector.bias', 'pipe.dit.blocks.1.self_attn.q.weight', 'pipe.dit.blocks.1.self_attn.q.bias', 'pipe.dit.blocks.1.self_attn.k.weight', 'pipe.dit.blocks.1.self_attn.k.bias', 'pipe.dit.blocks.1.self_attn.v.weight', 'pipe.dit.blocks.1.self_attn.v.bias', 'pipe.dit.blocks.1.self_attn.o.weight', 'pipe.dit.blocks.1.self_attn.o.bias', 'pipe.dit.blocks.1.self_attn.norm_q.weight', 'pipe.dit.blocks.1.self_attn.norm_k.weight', 'pipe.dit.blocks.1.camera_projector.weight', 'pipe.dit.blocks.1.camera_projector.bias', 'pipe.dit.blocks.2.self_attn.q.weight', 'pipe.dit.blocks.2.self_attn.q.bias', 'pipe.dit.blocks.2.self_attn.k.weight', 'pipe.dit.blocks.2.self_attn.k.bias', 'pipe.dit.blocks.2.self_attn.v.weight', 'pipe.dit.blocks.2.self_attn.v.bias', 'pipe.dit.blocks.2.self_attn.o.weight', 'pipe.dit.blocks.2.self_attn.o.bias', 'pipe.dit.blocks.2.self_attn.norm_q.weight', 'pipe.dit.blocks.2.self_attn.norm_k.weight', 'pipe.dit.blocks.2.camera_projector.weight', 'pipe.dit.blocks.2.camera_projector.bias', 'pipe.dit.blocks.3.self_attn.q.weight', 'pipe.dit.blocks.3.self_attn.q.bias', 'pipe.dit.blocks.3.self_attn.k.weight', 'pipe.dit.blocks.3.self_attn.k.bias', 'pipe.dit.blocks.3.self_attn.v.weight', 'pipe.dit.blocks.3.self_attn.v.bias', 'pipe.dit.blocks.3.self_attn.o.weight', 'pipe.dit.blocks.3.self_attn.o.bias', 'pipe.dit.blocks.3.self_attn.norm_q.weight', 'pipe.dit.blocks.3.self_attn.norm_k.weight', 'pipe.dit.blocks.3.camera_projector.weight', 'pipe.dit.blocks.3.camera_projector.bias', 'pipe.dit.blocks.4.self_attn.q.weight', 'pipe.dit.blocks.4.self_attn.q.bias', 'pipe.dit.blocks.4.self_attn.k.weight', 'pipe.dit.blocks.4.self_attn.k.bias', 'pipe.dit.blocks.4.self_attn.v.weight', 'pipe.dit.blocks.4.self_attn.v.bias', 'pipe.dit.blocks.4.self_attn.o.weight', 'pipe.dit.blocks.4.self_attn.o.bias', 'pipe.dit.blocks.4.self_attn.norm_q.weight', 'pipe.dit.blocks.4.self_attn.norm_k.weight', 'pipe.dit.blocks.4.camera_projector.weight', 'pipe.dit.blocks.4.camera_projector.bias', 'pipe.dit.blocks.5.self_attn.q.weight', 'pipe.dit.blocks.5.self_attn.q.bias', 'pipe.dit.blocks.5.self_attn.k.weight', 'pipe.dit.blocks.5.self_attn.k.bias', 'pipe.dit.blocks.5.self_attn.v.weight', 'pipe.dit.blocks.5.self_attn.v.bias', 'pipe.dit.blocks.5.self_attn.o.weight', 'pipe.dit.blocks.5.self_attn.o.bias', 'pipe.dit.blocks.5.self_attn.norm_q.weight', 'pipe.dit.blocks.5.self_attn.norm_k.weight', 'pipe.dit.blocks.5.camera_projector.weight', 'pipe.dit.blocks.5.camera_projector.bias', 'pipe.dit.blocks.6.self_attn.q.weight', 'pipe.dit.blocks.6.self_attn.q.bias', 'pipe.dit.blocks.6.self_attn.k.weight', 'pipe.dit.blocks.6.self_attn.k.bias', 'pipe.dit.blocks.6.self_attn.v.weight', 'pipe.dit.blocks.6.self_attn.v.bias', 'pipe.dit.blocks.6.self_attn.o.weight', 'pipe.dit.blocks.6.self_attn.o.bias', 'pipe.dit.blocks.6.self_attn.norm_q.weight', 'pipe.dit.blocks.6.self_attn.norm_k.weight', 'pipe.dit.blocks.6.camera_projector.weight', 'pipe.dit.blocks.6.camera_projector.bias', 'pipe.dit.blocks.7.self_attn.q.weight', 'pipe.dit.blocks.7.self_attn.q.bias', 'pipe.dit.blocks.7.self_attn.k.weight', 'pipe.dit.blocks.7.self_attn.k.bias', 'pipe.dit.blocks.7.self_attn.v.weight', 'pipe.dit.blocks.7.self_attn.v.bias', 'pipe.dit.blocks.7.self_attn.o.weight', 'pipe.dit.blocks.7.self_attn.o.bias', 'pipe.dit.block
+  warnings.warn(
+
+================================================================================
+TRAINABLE PARAMETERS VERIFICATION
+================================================================================
+
+Trainable modules (95 module groups):
+
+  📦 pipe.dit.blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+     pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.dense_target_control_encoder.residual_blocks:
+     Total params: 110,720
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 1.conv1.weight: 27,648
+         ├─ 1.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+         ├─ 1.conv1.bias: 32
+         ├─ 1.conv2.bias: 32
+
+  📦 pipe.dit._camera_patchify_conv:
+     Total params: 36,870
+         ├─ weight: 36,864
+         ├─ bias: 6
+
+  📦 pipe.dit._camera_plucker_patchify_encoder:
+     Total params: 35,840
+         ├─ weight: 30,720
+         ├─ bias: 5,120
+
+  📦 pipe.dit.control_patch_embedding:
+     Total params: 332,800
+         ├─ weight: 327,680
+         ├─ bias: 5,120
+
+  📦 pipe.dit.domain_embedding:
+     Total params: 10,240
+     pipe.dit.domain_embedding: 10,240
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+     pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_adapter.residual_blocks:
+     Total params: 55,360
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         ├─ 16.weight: 27,013,120
+         ├─ 17.weight: 27,013,120
+         ├─ 18.weight: 27,013,120
+         ├─ 19.weight: 27,013,120
+         ├─ 2.weight: 27,013,120
+         ├─ 20.weight: 27,013,120
+         ├─ 21.weight: 27,013,120
+         ├─ 22.weight: 27,013,120
+         ├─ 23.weight: 27,013,120
+         ├─ 24.weight: 27,013,120
+         ├─ 25.weight: 27,013,120
+         ├─ 26.weight: 27,013,120
+         ├─ 27.weight: 27,013,120
+         ├─ 28.weight: 27,013,120
+         ├─ 29.weight: 27,013,120
+         ├─ 3.weight: 27,013,120
+         ├─ 30.weight: 27,013,120
+         ├─ … (55 more)
+
+────────────────────────────────────────────────────────────────────────────────
+NOTE: FSDP is active; counts below are from the rank-local post-wrap parameter view, not a full global parameter census.
+Total trainable parameters: 6,325,796,530 (6325.80M)
+Total frozen parameters:    10,363,571,200 (10363.57M)
+Percentage trainable (overall): 37.90%
+Percentage trainable (in requested modules: dit): 33.60%
+================================================================================
+
+[torch-profiler] rank0 enabled (wait=3, warmup=1, active=5, repeat=1) -> /lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/torch_profiler
+Epoch 1/200:   0%|          | 29/7813 [08:21<37:24:54, 17.30s/it, opt_step=2559, loss=0.0673]  
+[Track4D-360] First train batch runtime flags: use_gradient_checkpointing=True, use_gradient_checkpointing_offload=False
+[soft_splat_warp] WARN: tracks grid 160x294 (N=47040) != image grid 288x512 (N=147456); bilinear-resampling tracks_world to image grid. This warning fires once per process.
+[track4d-runtime] rank0 patched DiTBlock.forward saw mode=track block_idx=0
+[Attention] Using: Flash Attention 2 (flash_attn)
+
+[Training] Reached max_train_steps=2560. Stopping training.
+
+[Checkpoint] Saving 597 parameters (total 6,325,796,530 ≈ 6325.80M):
+
+  📦 blocks.* — 40 blocks [0..39]:
+     Params per block: 131,107,840 × 40 = 5,244,313,600
+     Submodules (example from block 0):
+       • self_attn: 104,888,320
+       • camera_projector: 26,219,520
+
+  📦 _camera_patchify_conv.bias:
+     Total params: 6
+         _camera_patchify_conv.bias: 6
+
+  📦 _camera_patchify_conv.weight:
+     Total params: 36,864
+         _camera_patchify_conv.weight: 36,864
+
+  📦 _camera_plucker_patchify_encoder.bias:
+     Total params: 5,120
+         _camera_plucker_patchify_encoder.bias: 5,120
+
+  📦 _camera_plucker_patchify_encoder.weight:
+     Total params: 30,720
+         _camera_plucker_patchify_encoder.weight: 30,720
+
+  📦 control_patch_embedding.bias:
+     Total params: 5,120
+         control_patch_embedding.bias: 5,120
+
+  📦 control_patch_embedding.weight:
+     Total params: 327,680
+         control_patch_embedding.weight: 327,680
+
+  📦 domain_embedding:
+     Total params: 10,240
+         domain_embedding: 10,240
+
+  📦 pipe.dense_target_control_encoder.conv3d_patchify:
+     Total params: 10,272
+         ├─ weight: 10,240
+         ├─ bias: 32
+
+  📦 pipe.dense_target_control_encoder.null_token:
+     Total params: 156
+         pipe.dense_target_control_encoder.null_token: 156
+
+  📦 pipe.dense_target_control_encoder.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.dense_target_control_encoder.residual_blocks:
+     Total params: 110,720
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 1.conv1.weight: 27,648
+         ├─ 1.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+         ├─ 1.conv1.bias: 32
+         ├─ 1.conv2.bias: 32
+
+  📦 pipe.track_adapter.conv3d_patchify:
+     Total params: 65,568
+         ├─ weight: 65,536
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.mlp:
+     Total params: 80,028
+         ├─ 0.weight: 30,888
+         ├─ 2.weight: 24,336
+         ├─ 4.weight: 24,336
+         ├─ 0.bias: 156
+         ├─ 2.bias: 156
+         ├─ 4.bias: 156
+
+  📦 pipe.track_adapter.null_token:
+     Total params: 156
+         pipe.track_adapter.null_token: 156
+
+  📦 pipe.track_adapter.proj_down:
+     Total params: 5,024
+         ├─ weight: 4,992
+         ├─ bias: 32
+
+  📦 pipe.track_adapter.proj_up:
+     Total params: 5,148
+         ├─ weight: 4,992
+         ├─ bias: 156
+
+  📦 pipe.track_adapter.residual_blocks:
+     Total params: 55,360
+         ├─ 0.conv1.weight: 27,648
+         ├─ 0.conv2.weight: 27,648
+         ├─ 0.conv1.bias: 32
+         ├─ 0.conv2.bias: 32
+
+  📦 pipe.track_block_injector.encoders:
+     Total params: 1,080,729,600
+         ├─ 0.weight: 27,013,120
+         ├─ 1.weight: 27,013,120
+         ├─ 10.weight: 27,013,120
+         ├─ 11.weight: 27,013,120
+         ├─ 12.weight: 27,013,120
+         ├─ 13.weight: 27,013,120
+         ├─ 14.weight: 27,013,120
+         ├─ 15.weight: 27,013,120
+         … (72 more)

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/files/requirements.txt

@@ -0,0 +1,208 @@
+videox-fun==1.0.1
+track4d-360==0.1.0
+func_timeout==4.3.5
+click==8.3.2
+pip==26.0.1
+nvidia-cuda-nvrtc-cu12==12.6.77
+ImageIO==2.37.3
+numpy==1.26.4
+boto3==1.42.91
+albucore==0.0.24
+lpips==0.1.4
+torchsde==0.2.6
+tomesd==0.1.3
+yarl==1.23.0
+typer==0.24.1
+triton==3.5.0
+cycler==0.12.1
+multidict==6.7.1
+OpenEXR==3.4.10
+soxr==1.0.0
+kiwisolver==1.5.0
+simsimd==6.5.16
+parso==0.8.6
+einops==0.8.2
+ftfy==6.3.1
+platformdirs==4.9.6
+groovy==0.1.2
+nvidia-nvtx-cu12==12.6.77
+sentry-sdk==2.58.0
+certifi==2026.2.25
+markdown-it-py==4.0.0
+uvicorn==0.38.0
+nvidia-cudnn-cu12==9.10.2.21
+natsort==8.4.0
+awscrt==0.31.2
+accelerate==1.13.0
+deepspeed==0.18.9
+matplotlib==3.10.8
+mdurl==0.1.2
+regex==2026.4.4
+nvidia-cuda-cupti-cu12==12.6.80
+shellingham==1.5.4
+joblib==1.5.3
+zipp==3.23.1
+ipython_pygments_lexers==1.1.1
+executing==2.2.1
+safetensors==0.7.0
+typing-inspection==0.4.2
+httpcore==1.0.9
+semantic-version==2.10.0
+wandb==0.26.0
+asttokens==3.0.1
+datasets==4.8.4
+decorator==5.2.1
+attrs==26.1.0
+six==1.17.0
+importlib_metadata==9.0.0
+py-cpuinfo==9.0.0
+requests==2.33.1
+xxhash==3.6.0
+pytorch-lightning==2.6.1
+fsspec==2026.2.0
+albumentations==2.0.8
+soundfile==0.13.1
+torchaudio==2.9.0
+torchmetrics==1.9.0
+nvidia-cusolver-cu12==11.7.1.2
+scipy==1.17.1
+jmespath==1.1.0
+threadpoolctl==3.6.0
+plyfile==1.1.2
+peft==0.19.1
+pycparser==3.0
+Markdown==3.10.2
+aiosignal==1.4.0
+pydantic==2.13.2
+decord2==3.3.0
+safehttpx==0.1.7
+scikit-image==0.26.0
+fonttools==4.62.1
+lazy-loader==0.5
+smmap==5.0.3
+starlette==1.0.0
+Werkzeug==3.1.8
+propcache==0.4.1
+tensorboard==2.20.0
+rich==15.0.0
+modelscope==1.36.3
+absl-py==2.4.0
+traitlets==5.14.3
+networkx==3.6.1
+hf-xet==1.4.3
+h5py==3.16.0
+PyYAML==6.0.3
+Jinja2==3.1.6
+nvidia-cusparse-cu12==12.5.4.2
+idna==3.11
+transformers==4.46.2
+httpx==0.28.1
+MarkupSafe==3.0.3
+tensorboard-data-server==0.7.2
+pooch==1.9.0
+torch==2.9.0+cu126
+soupsieve==2.8.3
+numba==0.65.0
+torchdiffeq==0.2.5
+jedi==0.19.2
+charset-normalizer==3.4.7
+annotated-doc==0.0.4
+gradio==6.12.0
+Pygments==2.20.0
+pandas==3.0.2
+stack-data==0.6.3
+contourpy==1.3.3
+tomlkit==0.14.0
+scikit-learn==1.8.0
+librosa==0.11.0
+nvidia-cusparselt-cu12==0.7.1
+pytz==2026.1.post1
+go-with-the-track==0.1.0
+yunchang==0.6.4
+onnxruntime==1.24.4
+ninja==1.13.0
+sentencepiece==0.2.1
+pexpect==4.9.0
+multiprocess==0.70.19
+omegaconf==2.3.0
+h11==0.16.0
+pydub==0.25.1
+nvidia-cufft-cu12==11.3.0.4
+xfuser==0.4.5
+tifffile==2026.3.3
+msgpack==1.1.2
+psutil==7.2.2
+ipython==9.12.0
+hf-gradio==0.4.1
+diffusers==0.33.1
+python-dateutil==2.9.0.post0
+orjson==3.11.8
+huggingface_hub==0.36.2
+nvidia-cuda-runtime-cu12==12.6.77
+beautifulsoup4==4.14.3
+ptyprocess==0.7.0
+protobuf==7.34.1
+GitPython==3.1.46
+sympy==1.14.0
+packaging==26.1
+aiohttp==3.13.5
+nvidia-nvshmem-cu12==3.3.20
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+gradio_client==2.4.1
+stringzilla==4.6.0
+flatbuffers==25.12.19
+filelock==3.28.0
+imageio-ffmpeg==0.6.0
+fastapi==0.136.0
+nvidia-nvjitlink-cu12==12.6.85
+s3transfer==0.16.0
+python-multipart==0.0.26
+DistVAE==0.0.0b5
+urllib3==2.6.3
+nvidia-nccl-cu12==2.27.5
+av==15.1.0
+pure_eval==0.2.3
+wheel==0.46.3
+dill==0.4.1
+flash_attn==2.8.3
+track4d-360==0.1.0
+opencv-python==4.11.0.86
+anyio==4.13.0
+pytest==9.0.3
+nvidia-cufile-cu12==1.11.1.6
+lightning-utilities==0.15.3
+pluggy==1.6.0
+grpcio==1.80.0
+cffi==2.0.0
+iniconfig==2.3.0
+videox-fun==1.0.1
+matplotlib-inline==0.2.1
+tokenizers==0.20.3
+pydantic_core==2.46.2
+aiohappyeyeballs==2.6.1
+typing_extensions==4.15.0
+timm==1.0.26
+pyarrow==23.0.1
+mpmath==1.3.0
+prompt_toolkit==3.0.52
+mediapy==1.2.6
+llvmlite==0.47.0
+opencv-python-headless==4.11.0.86
+hjson==3.1.0
+tqdm==4.67.3
+torchvision==0.24.0
+pyparsing==3.3.2
+nvidia-cublas-cu12==12.6.4.1
+botocore==1.42.91
+nanobind==2.9.2
+wcwidth==0.6.0
+gitdb==4.0.12
+frozenlist==1.8.0
+brotli==1.2.0
+trampoline==0.1.2
+pillow==12.2.0
+audioread==3.1.0
+lightning==2.6.1
+setuptools==82.0.1
+nvidia-curand-cu12==10.3.7.77

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/files/wandb-metadata.json

@@ -0,0 +1,235 @@
+{
+  "os": "Linux-6.4.0-150600.23.73_15.0.14-cray_shasta_c_64k-aarch64-with-glibc2.38",
+  "python": "CPython 3.12.13",
+  "startedAt": "2026-06-02T09:15:02.848946Z",
+  "args": [
+    "--model_id_with_origin_paths",
+    "Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.1-T2V-14B:/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/Wan2.1_VAE.pth",
+    "--tokenizer_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/google/umt5-xxl",
+    "--task",
+    "sft:v2v",
+    "--model_variant",
+    "vt2v",
+    "--v2v_concat_mode",
+    "width",
+    "--height",
+    "288",
+    "--width",
+    "512",
+    "--num_frames",
+    "49",
+    "--track4d360_data_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/mixed_synth_recammaster_pair1x49_49f_288x512_ismb_nextstage.yaml",
+    "--steps_per_epoch",
+    "25000",
+    "--num_epochs",
+    "200",
+    "--learning_rate",
+    "2e-5",
+    "--gradient_accumulation_steps",
+    "1",
+    "--save_steps",
+    "500",
+    "--remove_prefix_in_ckpt",
+    "pipe.dit.",
+    "--trainable_models",
+    "dit.domain_embedding,dit.control_patch_embedding",
+    "--dit_finetune_modules",
+    "self_attn",
+    "--track_injection_mode",
+    "per_block",
+    "--track_injection_block_mode",
+    "concat_project",
+    "--track_config",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/config/track4d_360/default_conv3d_patchify_srcdepth.yaml",
+    "--train_sigma_shift",
+    "5.0",
+    "--use_query_frame_impulse_condition",
+    "--use_plucker_camera_control",
+    "--enable_v2v_plucker_camera_control",
+    "--use_plucker_raymap_tokens",
+    "--use_perlayer_camera_control",
+    "--camera_injection_mode",
+    "projector",
+    "--camera_plucker_encoder_mode",
+    "patchify",
+    "--use_dense_branch",
+    "--dense_in_channels",
+    "5",
+    "--dense_proj_dim",
+    "32",
+    "--dense_num_residual_blocks",
+    "2",
+    "--dense_alpha_track",
+    "1.0",
+    "--dense_conditioning_noise_max_level",
+    "0.15",
+    "--source_mask_boundary_depth_dropout_prob",
+    "0.5",
+    "--source_mask_boundary_kernel_size",
+    "5",
+    "--dense_warp_mode",
+    "hybrid_softsplat",
+    "--dense_warp_motion_threshold",
+    "0.005",
+    "--cond_drop_geo_prob",
+    "0.1",
+    "--cond_drop_track_prob",
+    "0.1",
+    "--cond_drop_dense_prob",
+    "0.0",
+    "--initial_step",
+    "2530",
+    "--output_path",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout",
+    "--dataset_num_workers",
+    "7",
+    "--dataloader_prefetch_factor",
+    "2",
+    "--drop_text_prob",
+    "0.1",
+    "--debug_latents_every",
+    "500",
+    "--allow_existing_checkpoints",
+    "--wandb_project",
+    "track4d-360",
+    "--wandb_name",
+    "ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp",
+    "--wandb_tags",
+    "ismb,track4d360,14b,per_block,conv3d,hybrid_dense,plucker_raymap_tokens,mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptoken,49f,288x512,concat_project,attn,cond_dropout,32gpu,8node,dense_noise_aug",
+    "--use_wandb",
+    "--vxf_init_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/weights/Wan2.1-T2V-14B/diffusion_pytorch_model*.safetensors",
+    "--use_gradient_checkpointing",
+    "--resume_checkpoint",
+    "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_cond_dropout/step-2530.safetensors",
+    "--max_train_steps",
+    "2560",
+    "--torch_profile_active_steps",
+    "5",
+    "--torch_profile_wait_steps",
+    "3",
+    "--torch_profile_warmup_steps",
+    "1",
+    "--torch_profile_repeat",
+    "1",
+    "--no_torch_profile_record_shapes",
+    "--no_torch_profile_memory"
+  ],
+  "program": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360/scripts/wan2.1_fun/train_track4d360.py",
+  "codePath": "scripts/wan2.1_fun/train_track4d360.py",
+  "codePathLocal": "scripts/wan2.1_fun/train_track4d360.py",
+  "git": {
+    "remote": "git@github.com:NIRVANALAN/track4d-360.git",
+    "commit": "e47380b0c0f4cbb87efcbad540b96a404c2ccc88"
+  },
+  "email": "yushi.lan@outlook.com",
+  "root": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/logs/track4d-360/ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout",
+  "host": "nid011121",
+  "executable": "/projects/public/u6ew/yslan_public/yslan_project/miniforge3/envs/th29py312/bin/python3.12",
+  "cpu_count": 288,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA GH200 120GB",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "255795724288",
+      "used": "549191680"
+    }
+  },
+  "memory": {
+    "total": "919613407232"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-d655252a-055e-6123-1fcc-96c384e94cb6"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-f32c462b-35e6-cd39-3ab1-ef2d12935745"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-c6448e24-bbf3-db8b-e6c7-7becaf680d11"
+    },
+    {
+      "name": "NVIDIA GH200 120GB",
+      "memoryTotal": "102625181696",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-2c148b70-ecf2-fb15-969b-cbb11a8b9bc4"
+    }
+  ],
+  "cudaVersion": "12.7",
+  "slurm": {
+    "cluster_name": "gracehopper",
+    "conf": "/var/spool/slurmd/conf-cache/slurm.conf",
+    "cpu_bind": "quiet,mask_cpu:0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_list": "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+    "cpu_bind_type": "mask_cpu:",
+    "cpu_bind_verbose": "quiet",
+    "cpus_on_node": "288",
+    "distribution": "cyclic",
+    "gpus": "32",
+    "gpus_on_node": "4",
+    "gpus_per_node": "4",
+    "gtids": "0",
+    "job_account": "brics.u6ew",
+    "job_cpus_per_node": "288(x8)",
+    "job_end_time": "1780398532",
+    "job_gid": "1483804847",
+    "job_gpus": "0,1,2,3",
+    "job_id": "4984787",
+    "job_name": "14b_pair1x49_profile_patchifylinear_fsdp2",
+    "job_nodelist": "nid[011121-011122,011125,011127-011129,011165,011167]",
+    "job_num_nodes": "8",
+    "job_partition": "workq",
+    "job_qos": "normal",
+    "job_start_time": "1780391332",
+    "job_uid": "1483804847",
+    "job_user": "yushi.u6ew",
+    "jobid": "4984787",
+    "launch_node_ipaddr": "10.242.4.57",
+    "localid": "0",
+    "mpi_type": "pmi2",
+    "nnodes": "8",
+    "nodeid": "0",
+    "nodelist": "nid[011121-011122,011125,011127-011129,011165,011167]",
+    "nprocs": "8",
+    "ntasks": "8",
+    "ntasks_per_node": "1",
+    "oom_kill_step": "0",
+    "prio_process": "0",
+    "procid": "0",
+    "srun_comm_host": "10.242.4.57",
+    "srun_comm_port": "45551",
+    "step_gpus": "0,1,2,3",
+    "step_id": "0",
+    "step_launcher_port": "45551",
+    "step_nodelist": "nid[011121-011122,011125,011127-011129,011165,011167]",
+    "step_num_nodes": "8",
+    "step_num_tasks": "8",
+    "step_resv_ports": "27631-27632",
+    "step_tasks_per_node": "1(x8)",
+    "stepid": "0",
+    "submit_dir": "/lus/lfs1aip2/projects/public/u6ew/yslan_public/yslan_project/track4d-360",
+    "submit_host": "login40",
+    "task_pid": "236308",
+    "tasks_per_node": "1(x8)",
+    "topology_addr": "global.group12.nid011121",
+    "topology_addr_pattern": "switch.switch.node",
+    "umask": "0022"
+  },
+  "writerId": "f0ztrk8t2qi0av8h7ptk8q57xoxkau1o"
+}

ismb_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout_288x512x49_14b_32gpu_fsdp/train/Wan2.1-Fun-14B_track4d360_hybrid_dense_plucker_mixed_synth_recammaster_pair1x49_nextstage_softsplat_raymaptokens_eb32_288x512_profile_patchifylinear_cond_dropout/wandb/run-20260602_091502-iwxrpu9h/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/track/injector/block_01/weight_norm":71.73619842529297,"train/timing/unit/WanVideoUnit_LatentNoiseAugmentation_s":5.2716633945237845e-05,"train/track/injector/block_00/grad_norm":0.03125328943133354,"train/timing/unit/WanVideoUnit_WarpedAppearance_s":5.737947503803298e-05,"train/track/injector/block_25/grad_norm":0.00310494564473629,"train/track/injector/block_29/weight_norm":71.70838165283203,"train/track/injector/block_16/weight_norm":71.75886535644531,"train/track/injector/block_08/grad_norm":0.009170296601951122,"train/track/injector/block_06/grad_norm":0.009117234498262405,"train/track/injector/block_11/grad_norm":0.00943054910749197,"train/track/injector/block_15/weight_norm":71.72622680664062,"_step":2559,"train/track/injector/block_33/weight_norm":71.6600341796875,"train/timing/optimizer_s":0.03643082920461893,"train/timing/unit/WanVideoUnit_PromptEmbedder_s":0.00025716342497617006,"train/track/injector/block_34/weight_norm":71.64431762695312,"train/track/injector/block_31/weight_norm":71.70207977294922,"train/track/injector/block_20/grad_norm":0.004566177260130644,"train/timing/microsteps":1,"train/track/injector/block_39/grad_norm":0.00405696127563715,"train/timing/unit/WanVideoUnit_NoiseInitializer_s":0.00019639797392301261,"train/track/injector/block_01/grad_norm":0.011140394024550915,"train/track/injector/block_10/weight_norm":71.77755737304688,"_timestamp":1.780392214034626e+09,"train/cond_dropout/drop_tracks":0.09375,"train/track/injector/block_35/weight_norm":71.68328094482422,"train/timing/data_wait_s":0.0015823920257389545,"train/timing/backward_s":5.116768166888505,"train/timing/reduction_s":0.14721127692610025,"train/track/injector/block_26/weight_norm":71.73145294189453,"train/timing/unit/WanVideoUnit_TrackPointCondition_s":0.019847922027111053,"train/timing/unit/WanVideoUnit_Vista4DStreams_s":5.4018557420931756e-05,"train/track/injector/block_27/weight_norm":71.70172882080078,"train/track/injector/block_04/grad_norm":0.009377378039062023,"train/hbm/post_t5_allocated_gb":28.869243621826172,"train/track/injector/block_28/weight_norm":71.74332427978516,"train/track/injector/block_21/grad_norm":0.00487802317366004,"train/timing/step_total_s":11.40470616798848,"train/track/injector/block_32/grad_norm":0.0020946995355188847,"train/track/injector/block_31/grad_norm":0.0020167483016848564,"train/track/injector/block_02/grad_norm":0.010800544172525406,"train/track/injector/block_14/weight_norm":71.7539291381836,"train/track/injector/block_17/weight_norm":71.72591400146484,"train/track/injector/block_37/grad_norm":0.0018434051889926195,"train/track/injector/block_04/weight_norm":71.78131103515625,"train/timing/unit/WanVideoUnit_InputVideoEmbedder_XYZCondition_s":1.6512749195098877,"train/track/injector/block_36/grad_norm":0.0018603148637339473,"_wandb":{"runtime":526},"train/track/injector/block_19/weight_norm":71.72638702392578,"train/timing/unit/WanVideoUnit_TeaCache_s":2.1591491531580687e-05,"train/track/injector/block_03/grad_norm":0.010950344614684582,"train/track/injector/block_28/grad_norm":0.0027651339769363403,"train/track/injector/block_09/grad_norm":0.009453365579247475,"train/track/injector/block_05/grad_norm":0.009369931183755398,"train/track/injector/block_13/weight_norm":71.73658752441406,"train/timing/unit/WanVideoUnit_CameraControl_NVS_s":0.003380503272637725,"train/track/injector/block_26/grad_norm":0.002930134069174528,"train/track/injector/block_25/weight_norm":71.7306137084961,"train/track/injector/block_22/weight_norm":71.7159423828125,"train/timing/forward_s":5.396990212611854,"train/track/injector/block_14/grad_norm":0.008379348553717136,"train/t5/on_cuda":-1,"train/cond_dropout/drop_dense":0.03125,"train/track/injector/block_30/weight_norm":71.70831298828125,"train/track/injector/block_17/grad_norm":0.0065108006820082664,"train/track/injector/block_08/weight_norm":71.77717590332031,"train/epoch":0,"train/track/injector/block_02/weight_norm":71.7703857421875,"train/timing/dit_loss_fn_s":2.527306079864502,"train/track/injector/block_29/grad_norm":0.0027308855205774307,"train/track/injector/block_03/weight_norm":71.80191802978516,"train/track/injector/block_20/weight_norm":71.71338653564453,"train/hbm/reserved_gb":39.55208969116211,"train/track/injector/block_30/grad_norm":0.002292318269610405,"train/track/injector/block_18/grad_norm":0.006294068414717913,"train/track/injector/block_16/grad_norm":0.007425781339406967,"train/track/injector/block_00/weight_norm":71.62019348144531,"train/track/injector/block_11/weight_norm":71.75846862792969,"train/timing/unit/WanVideoUnit_ModeDropoutAugmentation_s":3.1350209610536695e-05,"train/track/injector/block_05/weight_norm":71.79116821289062,"train/track/injector/block_19/grad_norm":0.005807674024254084,"train/track/injector/weight_norm_max":71.80191802978516,"train/track/injector/block_06/weight_norm":71.78491973876953,"train/track/injector/block_07/grad_norm":0.00951184332370758,"train/track/injector/block_21/weight_norm":71.71271514892578,"train/timing/unit/WanVideoUnit_UnifiedSequenceParallel_s":2.87372458842583e-05,"train/track/injector/block_22/grad_norm":0.005098571069538593,"train/timing/unit/WanVideoUnit_TargetVideoEncoder_s":0.829370379447937,"train/track/injector/block_39/weight_norm":71.61335754394531,"train/loss":0.06727957725524902,"train/timing/unit/WanVideoUnit_DenseTargetControl_s":0.25093698501586914,"_runtime":526,"train/track/injector/block_27/grad_norm":0.0027274005115032196,"train/timing/other_s":0.7073056823574007,"train/track/injector/grad_norm_max":0.03125328943133354,"train/hbm/allocated_gb":3.4353647232055664,"train/track/injector/block_07/weight_norm":71.78191375732422,"train/track/injector/block_36/weight_norm":71.64971923828125,"train/track/injector/block_10/grad_norm":0.009506093338131905,"train/track/injector/block_37/weight_norm":71.64720916748047,"train/track/injector/block_23/grad_norm":0.004205186385661364,"train/timing/unit/WanVideoUnit_V2VConcatenate_s":0.0001357975124847144,"train/track/injector/block_32/weight_norm":71.66665649414062,"train/track/adapter/weight_norm":17.002626419067383,"train/track/injector/block_24/weight_norm":71.72339630126953,"train/track/injector/block_23/weight_norm":71.7166976928711,"train/track/adapter/grad_norm":0.0050230869092047215,"train/track/injector/block_38/weight_norm":71.60942077636719,"train/track/injector/block_34/grad_norm":0.0019225106807425618,"train/track/injector/weight_norm_mean":71.7211685180664,"train/track/injector/block_12/grad_norm":0.008436022326350212,"train/track/injector/block_13/grad_norm":0.007800602354109287,"train/track/injector/block_33/grad_norm":0.0021519330330193043,"train/timing/unit/WanVideoUnit_ShapeChecker_s":5.8914534747600555e-05,"train/hbm/post_t5_reserved_gb":28.94489097595215,"train/track/injector/grad_norm_mean":0.006389424670487642,"train/track/injector/block_38/grad_norm":0.0021260089706629515,"train/track/injector/block_24/grad_norm":0.00367747456766665,"train/track/injector/block_12/weight_norm":71.7568359375,"train/track/injector/block_09/weight_norm":71.78079223632812,"train/track/injector/block_35/grad_norm":0.0019217939116060734,"train/track/injector/block_15/grad_norm":0.00687079643830657,"train/timing/unit/WanVideoUnit_CfgMerger_s":1.3017634046263993e-05,"train/track/injector/block_18/weight_norm":71.7179183959961}