Repository navigation
Expand file tree
/
Copy patheval_generation.py
More file actions
5396 lines (4968 loc) · 251 KB
/
Copy patheval_generation.py
File metadata and controls
5396 lines (4968 loc) · 251 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Evaluate GAE flow generation and reconstruction.
Use ``scripts/demo/generate.py`` for one-image + prompt generation. This lower-level
entry point supports dataset manifests, long rollouts, video metrics, direct
geometry dumps, and PLY export. The default ``euler_v3`` sampler is the sampler
used by the released GAE checkpoints.
"""
from __future__ import annotations
import sys
from pathlib import Path as _Path
_REPO_ROOT = _Path(__file__).resolve().parents[2]
for _p in (str(_REPO_ROOT / "src"), str(_REPO_ROOT / "scripts" / "eval")):
if _p not in sys.path:
sys.path.insert(0, _p)
import argparse
import hashlib
import json
import math
import os
import random
import shutil
import sys
import time
from contextlib import contextmanager
os.environ.setdefault("TMPDIR", "/tmp")
# Use local-ssd for HuggingFace cache (fast NVMe) if available;
# copies from /tmp or downloads on first run, then instant on subsequent runs.
_LOCAL_HF_CACHE = "/local-ssd/hf_cache"
if os.path.isdir("/local-ssd"):
os.makedirs(_LOCAL_HF_CACHE, exist_ok=True)
# If HF models already cached in /tmp, migrate to local-ssd
_tmp_hf = "/tmp/xdg-cache/huggingface/hub"
if os.path.isdir(_tmp_hf) and not os.listdir(_LOCAL_HF_CACHE):
import shutil
for d in os.listdir(_tmp_hf):
src = os.path.join(_tmp_hf, d)
dst = os.path.join(_LOCAL_HF_CACHE, d)
if os.path.isdir(src) and not os.path.exists(dst):
shutil.copytree(src, dst, symlinks=True)
os.environ["HF_HOME"] = "/local-ssd/hf_home"
os.environ["HUGGINGFACE_HUB_CACHE"] = _LOCAL_HF_CACHE
else:
os.environ.setdefault("HF_HOME", "/tmp/xdg-cache")
os.environ.setdefault("HUGGINGFACE_HUB_CACHE", "/tmp/xdg-cache/huggingface/hub")
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
# Insert scripts/ before ROOT so sibling imports (eval_data, …) resolve
# even when PYTHONPATH contains another top-level ``scripts`` package that would
# shadow ``ROOT/scripts`` for ``from scripts.*`` imports.
sys.path.insert(0, _SCRIPTS_DIR)
sys.path.insert(0, ROOT)
sys.path.insert(0, os.path.join(ROOT, "src"))
_HF_TO_LOCAL_TEXT = {
"Qwen/Qwen3-0.6B": "pretrained_models/qwen3_0.6b",
}
def _resolve_text_model_path(model_name: str) -> str:
"""Resolve known HF text encoders to checked-in local model dirs when present."""
candidates = [os.environ.get("QWEN3_LOCAL_PATH", ""), model_name]
mapped = _HF_TO_LOCAL_TEXT.get(model_name)
if mapped:
candidates.append(mapped)
for raw in candidates:
if not raw:
continue
probes = [raw] if os.path.isabs(raw) else [
os.path.join(ROOT, raw),
os.path.join(os.getcwd(), raw),
]
for path in probes:
if os.path.isdir(path) and os.path.isfile(os.path.join(path, "config.json")):
return path
return model_name
def _cache_pretrained_dir_to_local_ssd(path: str, tag: str, timeout_s: float = 1800.0) -> str:
"""Copy a local pretrained directory to node-local SSD once and reuse it.
Loading large safetensors from a network filesystem makes every concurrent
eval worker re-read roughly 1.5 GB. A completion marker and exclusive lock
let one worker populate a local SSD copy while the others wait for it.
"""
if not (path and os.path.isdir(path)):
return path
cache_root = os.path.join(_eval_cache_base(), "_eval_cache")
if os.path.abspath(path).startswith(os.path.abspath(cache_root) + os.sep):
return path
source_files: list[tuple[str, str]] = []
for root, _, files in os.walk(path):
for name in files:
src = os.path.join(root, name)
source_files.append((src, os.path.relpath(src, path)))
if not source_files:
return path
digest = hashlib.sha1(os.path.abspath(path).encode()).hexdigest()[:12]
local_dir = os.path.join(cache_root, f"{tag}_{digest}")
done_path = os.path.join(local_dir, ".complete")
lock_path = f"{local_dir}.lock"
def _is_complete() -> bool:
if not os.path.isfile(done_path):
return False
try:
return all(
os.path.isfile(os.path.join(local_dir, rel))
and os.path.getsize(os.path.join(local_dir, rel)) == os.path.getsize(src)
for src, rel in source_files
)
except OSError:
return False
if _is_complete():
print(f" [cache] Using {local_dir}")
return local_dir
owns_lock = False
try:
os.makedirs(cache_root, exist_ok=True)
try:
lock_fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.close(lock_fd)
owns_lock = True
except FileExistsError:
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if _is_complete():
print(f" [cache] Using {local_dir}")
return local_dir
time.sleep(1.0)
print(f" [cache] timed out waiting for {tag}; using {path}")
return path
os.makedirs(local_dir, exist_ok=True)
print(f" [cache] {path} -> {local_dir} ...", flush=True)
for src, rel in source_files:
dst = os.path.join(local_dir, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
try:
valid = os.path.isfile(dst) and os.path.getsize(dst) == os.path.getsize(src)
except OSError:
valid = False
if valid:
continue
tmp = f"{dst}.tmp.{os.getpid()}"
shutil.copy2(src, tmp)
os.replace(tmp, dst)
with open(done_path, "w", encoding="utf-8") as handle:
handle.write(os.path.abspath(path))
print(" [cache] Qwen3 local copy ready", flush=True)
return local_dir
except OSError as exc:
print(f" [cache] {tag} local-ssd copy failed ({exc}); using {path}", flush=True)
return path
finally:
if owns_lock:
try:
os.unlink(lock_path)
except FileNotFoundError:
pass
def _to_container_or_empty(node) -> dict:
if node is None:
return {}
if OmegaConf.is_config(node):
return OmegaConf.to_container(node, resolve=True) or {}
if isinstance(node, dict):
return dict(node)
return {}
import numpy as np
import torch
import torch.nn.functional as F
from diffusers import AutoencoderKL, AutoencoderKLWan
from einops import rearrange
from PIL import Image
from omegaconf import OmegaConf
from stage1.da3 import DA3Backbone
from stage1.gae_codec import GAECodec, DA3_LEVEL_DIM
from stage1.da3_direct_codec import DA3DirectCodec
from stage2.transport.flow import (
convert_x_to_v,
sample_logit_normal_t,
shift_from_latent_dim,
)
from stage2.models.camera import compute_plucker_6d_per_token
from utils.model_utils import instantiate_from_config
from scripts.demo.trajectory_utils import synthesize_free_trajectory
from eval_data import (
DATASET_ROOTS,
DATASET_DEFAULT_INTERVAL,
collect_scenes,
collect_scenes_re10k,
load_scenes_from_manifest,
load_image_and_camera,
clear_scannetpp_frame_cache,
prefetch_scannetpp_frames,
tensor_to_numpy_img,
depth_to_numpy_video,
compute_metrics,
visualize_trajectory,
raw_no_cls_to_dpt_input,
raw_to_dpt_input,
decode_to_depth,
)
from utils.train_runtime import normalise_c2w_for_batch, intrinsic_to_K
from eval_reconstruction import (
decode_dpt,
_ray_to_numpy,
_rayconf_to_numpy,
)
from utils.camera_from_ray import recover_poses
# ── Latent normalization (per-channel scaling OR full-covariance whitening) ──
# When stats file contains "whiten" / "unwhiten" matrices (Σ^(-1/2) / Σ^(+1/2)),
# diffusion operates in the whitened ≈ N(0, I) space — see docs/notes/
# 2026-05-26-raev2-analysis.md ("Final verdict"). Falls back to per-channel.
def _norm_latent(z: torch.Tensor, mean: torch.Tensor, std: torch.Tensor,
whiten: torch.Tensor | None) -> torch.Tensor:
if whiten is None:
return (z - mean) / std
# einsum requires uniform dtype. Whitening (Σ^(-1/2)) is precision-sensitive
# for anisotropic latents (cond≈100), so always run the matmul in fp32 and
# cast the result back to z's dtype. This is also what makes the helper
# safe to call outside an autocast context.
out_dtype = z.dtype
out = torch.einsum("ij,bjhw->bihw", whiten.float(),
(z.float() - mean.float()))
return out.to(out_dtype)
def _denorm_latent(y: torch.Tensor, mean: torch.Tensor, std: torch.Tensor,
unwhiten: torch.Tensor | None) -> torch.Tensor:
if unwhiten is None:
return y * std + mean
# See _norm_latent: fp32 matmul + cast back to caller dtype.
out_dtype = y.dtype
out = (torch.einsum("ij,bjhw->bihw", unwhiten.float(), y.float())
+ mean.float())
return out.to(out_dtype)
def compute_ssim(img1, img2):
"""Compute SSIM between two (B, C, H, W) tensors in [0,1]."""
C1 = 0.01 ** 2
C2 = 0.03 ** 2
ks, sigma = 11, 1.5
coords = torch.arange(ks, dtype=img1.dtype, device=img1.device) - ks // 2
g = torch.exp(-(coords ** 2) / (2 * sigma ** 2))
g = g / g.sum()
window = (g.unsqueeze(0) * g.unsqueeze(1)).unsqueeze(0).unsqueeze(0)
pad = ks // 2
vals = []
for c in range(img1.shape[1]):
g1, p1 = img1[:, c:c+1], img2[:, c:c+1]
mu_g = F.conv2d(g1, window, padding=pad)
mu_p = F.conv2d(p1, window, padding=pad)
sig_gg = F.conv2d(g1 * g1, window, padding=pad) - mu_g * mu_g
sig_pp = F.conv2d(p1 * p1, window, padding=pad) - mu_p * mu_p
sig_gp = F.conv2d(g1 * p1, window, padding=pad) - mu_g * mu_p
ssim_map = ((2 * mu_g * mu_p + C1) * (2 * sig_gp + C2)) / \
((mu_g**2 + mu_p**2 + C1) * (sig_gg + sig_pp + C2))
vals.append(ssim_map.mean().item())
return float(np.mean(vals))
def _fast_load(path, **kwargs):
"""torch.load with local caching for S3-FUSE/EFS paths.
The cache key is hashed from the FULL source path (not just the basename):
different runs all name their checkpoints ``0008000.pt`` etc., so a
basename-only key silently served a stale checkpoint from another run
(e.g. a d64 ckpt loaded against a d128 config → state_dict size mismatch).
A size check additionally re-copies if the cached file is incomplete/stale.
"""
import hashlib
import subprocess
path = os.path.abspath(os.fspath(path))
if path.startswith("/threed-code/") or path.startswith("/efs/"):
cache_dir = os.path.join(_eval_cache_base(), "_eval_cache")
os.makedirs(cache_dir, exist_ok=True)
src = path
h = hashlib.md5(os.path.abspath(path).encode()).hexdigest()[:12]
local = os.path.join(cache_dir, f"{h}_{os.path.basename(path)}")
try:
src_stat = os.stat(src)
local_stat = os.stat(local)
# Size alone is insufficient: checkpoints are occasionally replaced
# in-place, and two torch archives can have the same byte length.
# A normal ``cp`` gives the destination a current mtime, so an older
# or equal source mtime means the cached copy still covers the source.
fresh = (
local_stat.st_size == src_stat.st_size
and local_stat.st_mtime_ns >= src_stat.st_mtime_ns
)
except OSError:
fresh = False
if not fresh:
print(f" [cache] {src} -> {local} ...", end=" ", flush=True)
# atomic: cp to a pid-unique tmp then rename, so concurrent shards
# on the same node never read a half-written file.
_tmp = f"{local}.tmp.{os.getpid()}"
subprocess.run(["cp", src, _tmp], check=True)
os.replace(_tmp, local)
print("done (%.1f GB)" % (os.path.getsize(local) / 1e9))
else:
print(f" [cache] Using {local}")
return torch.load(local, **kwargs)
return torch.load(path, **kwargs)
def _unwrap_codec_state(ckpt):
"""Release ckpts use ``codec``; research dumps use ``ema_vae`` / ``vae``."""
if not isinstance(ckpt, dict):
return ckpt
for key in ("ema_codec", "codec", "ema_vae", "vae", "model"):
inner = ckpt.get(key)
if isinstance(inner, dict) and any(torch.is_tensor(v) for v in inner.values()):
return inner
return ckpt
def _eval_cache_base() -> str:
"""Local scratch root for weight caches (callers append '/_eval_cache').
Falls back off ``/local-ssd`` so the distributed cache job also works in
environments that lack ``/local-ssd``: reading a ~1.6 GB safetensors
directly over S3-FUSE via mmap otherwise fails with
'incomplete metadata, file not fully covered'. Override with EVAL_CACHE_ROOT.
"""
root = os.environ.get("EVAL_CACHE_ROOT")
if not root:
root = "/local-ssd" if os.path.isdir("/local-ssd") else "/tmp"
return root
_HF_TO_LOCAL_DA3 = {
"depth-anything/DA3-LARGE-1.1": "pretrained_models/da3_large",
"depth-anything/DA3-Large": "pretrained_models/da3_large",
"depth-anything/DA3-Base": "pretrained_models/da3",
}
def _resolve_da3_encoder_path(pretrained_path: str, da3_weights_path: str | None = None) -> str:
"""Map HF repo ids to local pretrained_models/ when available (offline eval)."""
candidates: list[str] = []
if pretrained_path:
candidates.append(pretrained_path)
mapped = _HF_TO_LOCAL_DA3.get(pretrained_path)
if mapped:
candidates.append(mapped)
if da3_weights_path:
parent = os.path.dirname(da3_weights_path)
if parent:
candidates.append(parent)
for raw in candidates:
p = raw if os.path.isabs(raw) else os.path.join(ROOT, raw)
if os.path.isdir(p) and os.path.isfile(os.path.join(p, "config.json")):
if raw != pretrained_path:
print(f" [da3] encoder_pretrained_path: {pretrained_path!r} -> {p}")
return p
return pretrained_path
def _cache_da3_dir(path: str) -> str:
"""Mirror a local DA3 model directory to /local-ssd for fast from_pretrained.
``DepthAnything3.from_pretrained`` reads ``model.safetensors`` (~1.6 GB)
from the directory; on a network filesystem that read is the slow
"Loading weights from local directory" step. Copy once to local NVMe and reuse.
A big weight file already pre-cached by ``_cache_file`` (as
``<dirname>_<filename>``) is symlinked rather than re-copied.
"""
if not (path and os.path.isdir(path)):
return path # HF id / missing → leave as-is for offline fallback
import shutil
cache_root = os.path.join(_eval_cache_base(), "_eval_cache")
if path.startswith(cache_root):
return path
name = os.path.basename(path.rstrip("/"))
local = os.path.join(cache_root, name)
done = local + ".done"
if os.path.isdir(local) and os.path.isfile(done):
print(f" [cache] Using {local}")
return local
os.makedirs(local, exist_ok=True)
for fn in sorted(os.listdir(path)):
src = os.path.join(path, fn)
if not os.path.isfile(src):
continue
dst = os.path.join(local, fn)
if os.path.islink(dst) or os.path.isfile(dst):
continue
prior = os.path.join(cache_root, f"{name}_{fn}") # _cache_file naming
try:
reuse = os.path.isfile(prior) and os.path.getsize(prior) == os.path.getsize(src)
except OSError:
reuse = False
if reuse:
os.symlink(prior, dst)
else:
print(f" [cache] {src} -> {dst} ...", end=" ", flush=True)
_tmp = f"{dst}.tmp.{os.getpid()}"
shutil.copy2(src, _tmp)
os.replace(_tmp, dst)
print("done (%.0f MB)" % (os.path.getsize(dst) / 1e6))
open(done, "w").close()
print(f" [cache] da3 encoder dir -> {local}")
return local
def depth_to_point_map(depth, K):
"""Convert depth map to 3D point map in camera coordinates.
Args:
depth: (BV, H, W) depth values
K: (BV, 3, 3) camera intrinsic matrices
Returns:
pmap: (BV, H, W, 3) xyz in camera coordinates
"""
import torch as _torch
BV, H, W = depth.shape
device, dtype = depth.device, depth.dtype
v_coords, u_coords = _torch.meshgrid(
_torch.arange(H, device=device, dtype=dtype),
_torch.arange(W, device=device, dtype=dtype),
indexing="ij",
)
fx = K[:, 0, 0].view(BV, 1, 1)
fy = K[:, 1, 1].view(BV, 1, 1)
cx = K[:, 0, 2].view(BV, 1, 1)
cy = K[:, 1, 2].view(BV, 1, 1)
x = (u_coords.unsqueeze(0) - cx) * depth / (fx + 1e-8)
y = (v_coords.unsqueeze(0) - cy) * depth / (fy + 1e-8)
return _torch.stack([x, y, depth], dim=-1)
def save_pointcloud_ply(path, xyz, rgb):
"""Save a colored point cloud to PLY.
Args:
path: output .ply file path
xyz: (N, 3) world-space coordinates, float
rgb: (N, 3) colors in [0, 255], uint8
"""
import os as _os
_os.makedirs(_os.path.dirname(path) or ".", exist_ok=True)
N = xyz.shape[0]
header = (
"ply\n"
"format ascii 1.0\n"
f"element vertex {N}\n"
"property float x\n"
"property float y\n"
"property float z\n"
"property uchar red\n"
"property uchar green\n"
"property uchar blue\n"
"end_header\n"
)
with open(path, "w") as f:
f.write(header)
for i in range(N):
f.write(f"{xyz[i,0]:.6f} {xyz[i,1]:.6f} {xyz[i,2]:.6f} "
f"{rgb[i,0]} {rgb[i,1]} {rgb[i,2]}\n")
def _save_geom_npz(path: str, arrays: dict) -> None:
"""Write geometry NPZ robustly (fuse/NFS-safe via local temp)."""
import shutil
import tempfile
out_dir = os.path.dirname(os.path.abspath(path)) or "."
os.makedirs(out_dir, exist_ok=True)
tmp_dir = "/local-ssd" if os.path.isdir("/local-ssd") else None
fd, tmp = tempfile.mkstemp(suffix=".npz", dir=tmp_dir)
os.close(fd)
try:
try:
np.savez_compressed(tmp, **arrays)
except OSError:
np.savez(tmp, **arrays)
try:
os.replace(tmp, path)
except OSError:
# Cross-filesystem or fuse mounts may reject replace; copy then unlink.
shutil.copyfile(tmp, path)
finally:
if os.path.exists(tmp):
os.remove(tmp)
def _umeyama_sim3(src: np.ndarray, tgt: np.ndarray):
"""Estimate similarity transform: tgt ≈ scale * (src @ R.T) + t.
Args:
src, tgt: (N, 3) corresponding points in row-vector convention.
Returns:
scale, R (3, 3), t (3,)
"""
assert src.shape == tgt.shape and src.shape[1] == 3
n = src.shape[0]
if n < 3:
return 1.0, np.eye(3, dtype=np.float64), np.zeros(3, dtype=np.float64)
mu_s = src.mean(axis=0)
mu_t = tgt.mean(axis=0)
src_c = src - mu_s
tgt_c = tgt - mu_t
cov = (tgt_c.T @ src_c) / n
U, D, Vt = np.linalg.svd(cov)
S = np.eye(3, dtype=np.float64)
if np.linalg.det(U) * np.linalg.det(Vt) < 0:
S[2, 2] = -1.0
R = U @ S @ Vt
var_s = (src_c * src_c).sum() / n
scale = float(np.trace(np.diag(D) @ S) / var_s) if var_s > 1e-12 else 1.0
t = mu_t - scale * (R @ mu_s)
return scale, R, t
def _apply_sim3(xyz: np.ndarray, scale: float, R: np.ndarray, t: np.ndarray) -> np.ndarray:
return (scale * (xyz @ R.T)) + t
def _view_pointcloud_from_depth(
depth_hw: np.ndarray,
rgb_chw: np.ndarray,
c2w: np.ndarray,
K: np.ndarray,
*,
stride: int = 4,
max_depth: float = 100.0,
) -> tuple[np.ndarray, np.ndarray]:
"""Unproject one view to world-space points (subsampled grid)."""
H, W = depth_hw.shape
ys = np.arange(0, H, stride)
xs = np.arange(0, W, stride)
yy, xx = np.meshgrid(ys, xs, indexing="ij")
d = depth_hw[yy, xx].astype(np.float64)
valid = (d > 1e-3) & (d < max_depth)
if not np.any(valid):
return np.zeros((0, 3), dtype=np.float64), np.zeros((0, 3), dtype=np.uint8)
u = xx[valid].astype(np.float64)
v = yy[valid].astype(np.float64)
z = d[valid]
fx, fy = K[0, 0], K[1, 1]
cx, cy = K[0, 2], K[1, 2]
x = (u - cx) * z / (fx + 1e-8)
y = (v - cy) * z / (fy + 1e-8)
pts_cam = np.stack([x, y, z], axis=-1)
R_w = c2w[:3, :3]
t_w = c2w[:3, 3]
pts_w = (pts_cam @ R_w.T) + t_w
if rgb_chw.ndim == 3 and rgb_chw.shape[0] == 3:
rgb_hwc = np.transpose(rgb_chw, (1, 2, 0))
else:
rgb_hwc = rgb_chw
cols = (rgb_hwc[yy, xx][valid] * 255.0).clip(0, 255).astype(np.uint8)
return pts_w, cols
def _subsample_xyz(pts: np.ndarray, max_pts: int, rng: np.random.Generator) -> np.ndarray:
if pts.shape[0] <= max_pts:
return pts
idx = rng.choice(pts.shape[0], max_pts, replace=False)
return pts[idx]
def _bidirectional_chamfer(
a: np.ndarray,
b: np.ndarray,
device: torch.device,
*,
block: int = 2048,
) -> float:
"""Symmetric Chamfer (mean of both NN directions), chunked for VRAM."""
if a.shape[0] == 0 or b.shape[0] == 0:
return float("nan")
def _run(dev: torch.device) -> float:
ta = torch.from_numpy(a).float().to(dev)
tb = torch.from_numpy(b).float().to(dev)
def _nn_mean(src, tgt):
acc = 0.0
for i in range(0, src.shape[0], block):
d = torch.cdist(src[i : i + block], tgt)
acc += d.min(dim=1).values.sum().item()
return acc / src.shape[0]
return float(0.5 * (_nn_mean(ta, tb) + _nn_mean(tb, ta)))
try:
return _run(device)
except RuntimeError:
if device.type == "cuda":
torch.cuda.empty_cache()
return _run(torch.device("cpu"))
raise
def _median_scale_pair(a: np.ndarray, b: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
"""Scale both clouds so median ||p|| of the union is 1 (scene-invariant)."""
if a.shape[0] == 0 and b.shape[0] == 0:
return a, b
all_pts = np.concatenate([p for p in (a, b) if p.shape[0] > 0], axis=0)
norms = np.linalg.norm(all_pts, axis=-1)
valid = norms > 1e-6
if not np.any(valid):
return a, b
scale = float(np.median(norms[valid]))
if scale < 1e-8:
return a, b
return a / scale, b / scale
def compute_ref_tgt_pc_gap(
dpt_out,
depth_tensor,
rgb_imgs,
H: int,
W: int,
device,
*,
cond_num: int,
stride: int = 4,
max_pts: int = 30000,
rng: np.random.Generator | None = None,
) -> float:
"""Point-cloud discrepancy between ref views [:cond] and tgt views [cond:].
Used by tab:ref_ablation. Decodes poses from DPT rays, unprojects depth,
median-scales the scene, then reports bidirectional Chamfer.
"""
if cond_num <= 0:
return float("nan")
n_views = int(depth_tensor.shape[0])
if cond_num >= n_views:
return float("nan")
rng = rng or np.random.default_rng(0)
view_pcs = _recover_view_pointclouds(
dpt_out, depth_tensor, rgb_imgs, H, W, device, stride=stride,
)
ref = np.concatenate([view_pcs[i][0] for i in range(cond_num)], axis=0)
tgt = np.concatenate(
[view_pcs[i][0] for i in range(cond_num, n_views)], axis=0,
)
ref = ref[np.isfinite(ref).all(axis=1)]
tgt = tgt[np.isfinite(tgt).all(axis=1)]
ref, tgt = _median_scale_pair(ref, tgt)
ref = _subsample_xyz(ref, max_pts, rng)
tgt = _subsample_xyz(tgt, max_pts, rng)
return _bidirectional_chamfer(ref, tgt, device)
def _save_progressive_geometry(dpt_out, depth, rgb, path, H, W, stride, poses=None):
from scripts.demo.progressive_preview import save_geometry
ray = _ray_to_numpy(dpt_out["ray"])
if poses is None:
poses = recover_poses(ray, _rayconf_to_numpy(dpt_out.get("ray_conf")),
input_size=(H, W), return_per_view_intrinsics=True)
save_geometry(path, ray, depth.detach().float().cpu().numpy(),
rgb.detach().float().cpu().numpy(), *poses,
np.array([H, W]), stride=max(1, stride))
print(f" [preview] {path}", flush=True)
def _recover_view_pointclouds(
dpt_out, depth_tensor, rgb_imgs, H, W, device, *,
stride: int = 4,
) -> list[tuple[np.ndarray, np.ndarray]]:
"""Per-view world point clouds from the decoded DA3 ray/depth pair.
``recover_poses`` is still run on the decoded ray to keep the official DA3
camera-pose convention and validate the ray output. Point coordinates,
however, are built directly from that same ray: its first three channels
are world-frame directions and its last three channels are the per-view
camera center. Mixing the ray translation scale with a separate pinhole
back-projection from recovered ``K/c2w`` causes geoft point clouds to drift.
"""
ray = dpt_out.get("ray")
rc = dpt_out.get("ray_conf")
if ray is None:
raise RuntimeError("DPT output missing ray — cannot build point cloud")
ray_np = _ray_to_numpy(ray)
rc_np = _rayconf_to_numpy(rc)
# Keep the official ray-pose fit in the evaluation path. The point map
# below intentionally uses the ray's native origin/direction representation
# so all geometry terms share one metric gauge.
recover_poses(
ray_np, rc_np, input_size=(H, W),
return_per_view_intrinsics=True,
)
if ray_np.ndim != 4 or ray_np.shape[-1] != 6:
raise ValueError(f"expected ray shape (V,H,W,6), got {ray_np.shape}")
n_views, ray_h, ray_w = ray_np.shape[:3]
d_t = depth_tensor.float()
if d_t.ndim == 4:
d_t = d_t.squeeze(1)
if d_t.ndim != 3 or d_t.shape[0] != n_views:
raise ValueError(
f"depth/ray view mismatch: depth={tuple(d_t.shape)} ray={ray_np.shape}"
)
if d_t.shape[-2:] != (ray_h, ray_w):
d_t = F.interpolate(
d_t.unsqueeze(1), size=(ray_h, ray_w), mode="bilinear",
align_corners=False,
).squeeze(1)
d_hw = d_t.clamp(min=1e-3).cpu().numpy()
points = ray_np[..., 3:] + d_hw[..., None] * ray_np[..., :3]
if rgb_imgs is None:
rgb_np = np.zeros((n_views, 3, ray_h, ray_w), dtype=np.float32)
else:
rgb_t = rgb_imgs.float()
if rgb_t.ndim != 4 or rgb_t.shape[0] != n_views:
raise ValueError(
f"rgb/ray view mismatch: rgb={tuple(rgb_t.shape)} ray={ray_np.shape}"
)
if rgb_t.shape[-2:] != (ray_h, ray_w):
rgb_t = F.interpolate(
rgb_t, size=(ray_h, ray_w), mode="bilinear", align_corners=False,
)
rgb_np = rgb_t.cpu().numpy()
view_pcs: list[tuple[np.ndarray, np.ndarray]] = []
for vi in range(n_views):
xyz_v = points[vi][::stride, ::stride]
depth_v = d_hw[vi][::stride, ::stride]
valid = np.isfinite(xyz_v).all(axis=-1) & (depth_v > 1e-3) & (depth_v < 100.0)
xyz = xyz_v[valid].astype(np.float64, copy=False)
rgb_hwc = np.transpose(rgb_np[vi], (1, 2, 0))
cols = (rgb_hwc[::stride, ::stride][valid] * 255.0).clip(0, 255).astype(np.uint8)
view_pcs.append((xyz, cols))
return view_pcs
def _merge_overlap_aligned_pointclouds(
per_chunk_view_pcs: list[list[tuple[np.ndarray, np.ndarray]]],
roll_cond_num: int,
) -> tuple[np.ndarray, np.ndarray, list[float]]:
"""Align chunk-local PCs via overlap views (Sim3) and merge.
Chunk 0 defines the global frame. For chunk i>0, estimate Sim3 mapping
overlap views [0:roll) onto the last ``roll`` global views, apply to
the non-overlap views only, then append.
"""
if not per_chunk_view_pcs:
return np.zeros((0, 3)), np.zeros((0, 3), dtype=np.uint8), []
global_views: list[tuple[np.ndarray, np.ndarray]] = []
acc_xyz, acc_rgb = [], []
residuals: list[float] = []
for xyz, rgb in per_chunk_view_pcs[0]:
global_views.append((xyz, rgb))
acc_xyz.append(xyz)
acc_rgb.append(rgb)
for ci in range(1, len(per_chunk_view_pcs)):
chunk_pcs = per_chunk_view_pcs[ci]
R = roll_cond_num
if len(chunk_pcs) < R:
raise ValueError(f"chunk {ci} has {len(chunk_pcs)} views < roll {R}")
if len(global_views) < R:
raise ValueError("not enough global views for overlap alignment")
src_pts, tgt_pts = [], []
for r in range(R):
s_xyz, _ = chunk_pcs[r]
t_xyz, _ = global_views[-R + r]
n = min(len(s_xyz), len(t_xyz))
if n == 0:
continue
src_pts.append(s_xyz[:n])
tgt_pts.append(t_xyz[:n])
if not src_pts:
scale, Rm, t = 1.0, np.eye(3), np.zeros(3)
else:
src_all = np.concatenate(src_pts, axis=0)
tgt_all = np.concatenate(tgt_pts, axis=0)
scale, Rm, t = _umeyama_sim3(src_all, tgt_all)
pred = _apply_sim3(src_all, scale, Rm, t)
residuals.append(float(np.linalg.norm(pred - tgt_all, axis=1).mean()))
for vi in range(R, len(chunk_pcs)):
xyz, rgb = chunk_pcs[vi]
xyz_g = _apply_sim3(xyz, scale, Rm, t)
global_views.append((xyz_g, rgb))
acc_xyz.append(xyz_g)
acc_rgb.append(rgb)
xyz = np.concatenate(acc_xyz, axis=0)
rgb = np.concatenate(acc_rgb, axis=0)
# Preserve the concatenated per-view order. Randomly truncating a long
# rollout here prevents downstream progressive renders from recovering
# which frame each point came from.
return xyz, rgb, residuals
def _smooth_latents_temporal(z: torch.Tensor, strength: float) -> torch.Tensor:
"""Symmetric 3-frame smoothing over latent time/view axis."""
if strength <= 0 or z.shape[0] < 2:
return z
s = float(max(0.0, min(strength, 0.95)))
out = z.clone()
if z.shape[0] == 2:
out[0] = (1.0 - s) * z[0] + s * z[1]
out[1] = s * z[0] + (1.0 - s) * z[1]
return out
out[0] = (1.0 - s) * z[0] + s * z[1]
out[-1] = s * z[-2] + (1.0 - s) * z[-1]
out[1:-1] = (
(0.5 * s) * z[:-2]
+ (1.0 - s) * z[1:-1]
+ (0.5 * s) * z[2:]
)
return out
def _inject_rgb_decoder_from_ckpt(
vae_cfg: dict,
vae_sd: dict,
temporal_mode: str = "config",
) -> None:
"""Build/reconcile RGB decoder architecture with checkpoint keys."""
block_idxs = [
int(k.split(".")[2])
for k in vae_sd
if k.startswith("rgb_head.blocks.") and k.split(".")[2].isdigit()
]
if not block_idxs:
print(" [vae] ckpt has no rgb_head weights → RGB output disabled")
return
detected_depth = max(block_idxs) + 1
injected = "rgb_decoder" not in vae_cfg
if injected:
vae_cfg["rgb_decoder"] = dict(
hidden_dim=int(vae_cfg.get("rgb_decoder_hidden", 1024)),
depth=detected_depth,
num_heads=int(vae_cfg.get("rgb_decoder_heads", 16)),
ffn_ratio=4.0,
patch_size=14,
)
rgb_dec = vae_cfg["rgb_decoder"]
# A TimeSformer-style "divided" head keeps time inside every block
# (t_attn -> s_attn -> mlp) instead of interleaving separate
# rgb_head.temporal_blocks.*, so the interleaved probe below sees nothing and
# would rebuild it as a per-frame head. Detect it up front and pin the mode.
_divided_blocks = any(
k.startswith("rgb_head.blocks.") and ".t_attn." in k for k in vae_sd
)
if _divided_blocks:
if temporal_mode == "off":
raise RuntimeError(
"ckpt has a divided space-time rgb_head but RGB_HEAD_TEMPORAL=off "
"was requested; a per-frame decode of these weights is impossible.")
_proj_in = vae_sd.get("rgb_head.proj_in.weight")
if _proj_in is not None:
rgb_dec["hidden_dim"] = int(_proj_in.shape[0])
rgb_dec["depth"] = detected_depth
rgb_dec["temporal"] = True
rgb_dec["temporal_mode"] = "divided"
rgb_dec.setdefault("temporal_num_heads", int(rgb_dec.get("num_heads", 16)))
# Interleaved-only knobs; RGBHead ignores them in divided mode, but leaving
# them in the dict makes the printed config misleading.
for key in ("temporal_every", "temporal_max_views",
"temporal_ffn_ratio", "temporal_gated"):
rgb_dec.pop(key, None)
print(f" [vae] divided space-time RGBHead detected "
f"(depth={detected_depth}, hidden={rgb_dec.get('hidden_dim')}, "
f"heads={rgb_dec.get('num_heads')})")
if injected:
print(f" [vae] injecting rgb_decoder (auto-detected from ckpt)")
return
tblock_idxs = sorted({
int(k.split(".")[2])
for k in vae_sd
if k.startswith("rgb_head.temporal_blocks.")
and len(k.split(".")) > 2
and k.split(".")[2].isdigit()
})
if tblock_idxs:
temporal_every = (
tblock_idxs[1] - tblock_idxs[0]
if len(tblock_idxs) >= 2
else max(1, detected_depth)
)
has_ffn = any(
".ffn." in k
for k in vae_sd
if k.startswith("rgb_head.temporal_blocks.")
)
detected_temporal = dict(
temporal=True,
temporal_num_heads=int(rgb_dec.get("num_heads", 16)),
temporal_max_views=64,
temporal_every=temporal_every,
temporal_ffn_ratio=2.0 if has_ffn else 0.0,
)
else:
temporal_every = None
has_ffn = False
detected_temporal = {"temporal": False}
if temporal_mode == "auto" or (temporal_mode == "config" and injected):
for key in (
"temporal_num_heads",
"temporal_max_views",
"temporal_every",
"temporal_ffn_ratio",
):
rgb_dec.pop(key, None)
rgb_dec.update(detected_temporal)
elif temporal_mode == "off":
rgb_dec["temporal"] = False
elif temporal_mode == "on":
rgb_dec["temporal"] = True
if tblock_idxs:
rgb_dec.update(detected_temporal)
else:
rgb_dec.setdefault("temporal_num_heads", int(rgb_dec.get("num_heads", 16)))
rgb_dec.setdefault("temporal_max_views", 64)
rgb_dec.setdefault("temporal_every", 2)
rgb_dec.setdefault("temporal_ffn_ratio", 0.0)
elif temporal_mode != "config":
raise ValueError(f"Unknown RGB head temporal mode: {temporal_mode!r}")
temporal_enabled = bool(rgb_dec.get("temporal", False))
if temporal_enabled and tblock_idxs:
print(
f" [vae] temporal RGBHead enabled "
f"(blocks={tblock_idxs}, every={temporal_every}, ffn={has_ffn})"
)
else:
print(
f" [vae] temporal RGBHead {'enabled' if temporal_enabled else 'disabled'} "
f"(mode={temporal_mode}, ckpt_temporal={bool(tblock_idxs)})"
)
if injected:
print(
f" [vae] injecting rgb_decoder (depth={detected_depth} "
f"auto-detected from ckpt) so RGB head is built"
)
def _latent_chunk_to_view_pointclouds(
z_chunk, vae, rae, backbone_norm, rgb_imgs,
H, W, device, use_amp, amp_dtype, *,
stride: int = 4,
) -> list[tuple[np.ndarray, np.ndarray]]:
with torch.amp.autocast("cuda", enabled=use_amp, dtype=amp_dtype):
seq, h_l, w_l = vae._decode_trunk(z_chunk)
c_trunk = seq.shape[-1]
recon_raw = vae.dec_conv(
seq.permute(0, 2, 1).reshape(-1, c_trunk, h_l, w_l))
recon_feats = vae.denormalize_and_split(recon_raw)
n_views = z_chunk.shape[0]
gen_dpt_in = raw_no_cls_to_dpt_input(recon_feats, backbone_norm, n_views)
gen_dpt_out = decode_dpt(gen_dpt_in, rae.rae_cl_decoder, H, W)
depth = gen_dpt_out["depth"]
return _recover_view_pointclouds(
gen_dpt_out, depth, rgb_imgs, H, W, device, stride=stride,
)
def _scene_pointcloud_from_dpt(
dpt_out, depth_tensor, rgb_imgs, n_views, H, W, device, *,
stride: int = 4,