NVIDIA AI Introduce SpatialClaw: A Training-Free Agent That Treats Code as the Action Interface for Spatial Reasoning

git clone --recursive https://github.com/NVlabs/SpatialClaw.git
cd SpatialClaw
bash spatial_agent/scripts/setup.sh
cp .env.example .env        # add API keys, or self-host vLLM
python -m spatial_agent.entrypoints.run 
    --dataset spatial_agent/config/dataset/erqa.json 
    --model   spatial_agent/config/model/gemini-3-pro.json 
    --concurrency 4

# Reconstruct the scene, then segment both objects in one video pass
recon = tools.Reconstruct.Reconstruct(InputImages)
seg = tools.SAM3.segment_video_by_text(["radiator heater", "door"])
show(seg.visualize(1))                         # inspect the masks first

# Closest-point distance via KD-tree, not centroids
pts_h = seg.get_masked_points(recon, frame=1, object=0)   # object 0 = heater
pts_d = seg.get_masked_points(recon, frame=2, object=1)   # object 1 = door
dists, _ = scipy.spatial.KDTree(pts_d).query(pts_h, k=1)
ReturnAnswer(float(dists.min()))

Back</button>
<button class=”c primary” id=”sc-next”>Run next step </button>
<button class=”c” id=”sc-reset”>Reset</button>
<span class=”prog” id=”sc-prog”></span>
</div>

<div class=”foot”>
<span>Faithful to the paper’s walkthrough · interface logic is illustrative</span>
<span>Built for <b>Marktechpost</b> · verified Jun 2026</span>
</div>
</div>

<script>
(function(){
var root=document.getElementById(‘sc-root’);
if(!root)return;
var $=function(s){return root.querySelector(s)};

// — step data, faithful to Figure 2 of the SpatialClaw paper —
var DATA={
single:{
label:”single-pass · no persistence”,
stateNote:”No intermediate state. One complete program is committed before any execution feedback is seen.”,
vars:[],
steps:[{
think:”Write one complete program now, before seeing any mask, depth map, or error.”,
code:'<span class=”cm”># commit the full analysis up front</span>nrecon = tools.<span class=”fn”>Reconstruct</span>(frames_for_recon)nseg_heater = tools.<span class=”fn”>SAM3</span>(img_heater, <span class=”st”>”white radiator heater”</span>)nimg_door = InputImages[2]n<span class=”cm”># … compute centroid distance …</span>n<span class=”kw”>else</span>:n <span class=”fn”>ReturnAnswer</span>(<span class=”st”>”Could not determine distance”</span>)’,
fb:’Single run complete · no chance to inspect or revise.’,
final:true, answer:”1.638″, correct:false,
why:”Wrong — the validity of the mask was never checked.”
}]
},
struct:{
label:”named results only”,
stateNote:”Each step binds one named result. They cannot be freely composed with NumPy or SciPy at test time.”,
vars:[],
steps:[
{think:”Reconstruct the scene.”,
code:'{<span class=”st”>”tool”</span>: <span class=”st”>”Reconstruct”</span>,n <span class=”st”>”args”</span>: {<span class=”st”>”image”</span>: <span class=”st”>”InputImages[1]”</span>}}’,
fb:'<span class=”ok”>stored</span> → result_1 : Reconstruction’,
addVars:[{n:”result_1″,t:”Reconstruction”}]},
{think:”Segment the heater.”,
code:'{<span class=”st”>”tool”</span>: <span class=”st”>”SAM3″</span>,n <span class=”st”>”args”</span>: {<span class=”st”>”image”</span>: <span class=”st”>”InputImages[1]”</span>, <span class=”st”>”prompt”</span>: <span class=”st”>”heater”</span>}}’,
fb:'<span class=”ok”>stored</span> → result_2 : mask’,
addVars:[{n:”result_2″,t:”mask”}]},
{think:”Segment the door.”,
code:'{<span class=”st”>”tool”</span>: <span class=”st”>”SAM3″</span>,n <span class=”st”>”args”</span>: {<span class=”st”>”image”</span>: <span class=”st”>”InputImages[2]”</span>, <span class=”st”>”prompt”</span>: <span class=”st”>”door”</span>}}’,
fb:'<span class=”ok”>stored</span> → result_3 : mask’,
addVars:[{n:”result_3″,t:”mask”}]},
{think:”Measure distance with the predefined tool, between the two centroids.”,
code:'{<span class=”st”>”tool”</span>: <span class=”st”>”compute_dist”</span>,n <span class=”st”>”args”</span>: {<span class=”st”>”seg_1″</span>: <span class=”st”>”result_2″</span>, <span class=”st”>”seg_2″</span>: <span class=”st”>”result_3″</span>}}’,
fb:’Distance: 6.5 · no tool exists for the <em>closest</em> point’,
addVars:[{n:”result_4″,t:”float = 6.5″}]},
{think:”No registered tool returns the closest point. Submit the centroid distance.”,
code:'{<span class=”st”>”tool”</span>: <span class=”st”>”ReturnAnswer”</span>, <span class=”st”>”args”</span>: {<span class=”st”>”value”</span>: 6.5}}’,
fb:’answer submitted’,
final:true, answer:”6.5″, correct:false,
why:”Wrong — the schema has no tool for deriving the closest point.”}
]
},
claw:{
label:”persistent python kernel”,
stateNote:”Every object stays a live Python variable. The agent composes, inspects, and revises across steps.”,
vars:[],
steps:[
{think:”Reconstruct the scene, segment both objects, and render the masks to inspect them.”,
code:’seg_h = tools.<span class=”fn”>SAM3</span>(InputImages[1], <span class=”st”>”radiator”</span>)nseg_d = tools.<span class=”fn”>SAM3</span>(InputImages[2], <span class=”st”>”door”</span>)nrecon = tools.<span class=”fn”>Reconstruct</span>(InputImages[::100])n<span class=”fn”>show</span>(seg_h, seg_d) <span class=”cm”># visual feedback</span>’,
fb:'<span class=”ok”>Status: Success</span> · <span class=”show”>show() → mask overlay rendered for inspection</span>’,
addVars:[{n:”seg_h”,t:”PerFrameMask”},{n:”seg_d”,t:”PerFrameMask”},{n:”recon”,t:”Reconstruction”}]},
{think:”Masks verified. Compute each object’s 3D centroid and the distance between them.”,
code:’points_h = recon.points[seg_h]npoints_d = recon.points[seg_d]ncentroid_h = tools.<span class=”fn”>get_centroid</span>(points_h)ncentroid_d = tools.<span class=”fn”>get_centroid</span>(points_d)ndist = tools.<span class=”fn”>dist</span>(centroid_h, centroid_d)n<span class=”fn”>print</span>(<span class=”st”>f”Distance: {dist}”</span>)’,
fb:'<span class=”ok”>Status: Success</span> · Distance: 1.4807′,
addVars:[{n:”points_h”,t:”ndarray (12361,3)”},{n:”points_d”,t:”ndarray”},{n:”centroid_h”,t:”ndarray (3,)”},{n:”centroid_d”,t:”ndarray (3,)”},{n:”dist”,t:”float = 1.4807″}]},
{revise:true,
think:”Wait — tools.get_centroid uses the median, which is not the closest point. Switch to scipy.spatial.KDTree.”,
code:’tree_d = scipy.spatial.<span class=”fn”>KDTree</span>(points_d)ndists, _ = tree_d.<span class=”fn”>query</span>(points_h, k=1)nmin_dist = dists.<span class=”fn”>min</span>()n<span class=”fn”>print</span>(<span class=”st”>f”Min distance: {min_dist}”</span>)’,
fb:'<span class=”ok”>Status: Success</span> · Min distance: 0.9439′,
addVars:[{n:”tree_d”,t:”KDTree”},{n:”dists”,t:”ndarray (12361,)”},{n:”min_dist”,t:”float = 0.9439″}]},
{think:”Closest-point distance cross-checked against the visual evidence. Submit it.”,
code:'<span class=”fn”>ReturnAnswer</span>(0.9439)’,
fb:’answer submitted’,
final:true, answer:”0.9439″, correct:true,
why:”Correct — composition + intermediate verification reached the closest point.”}
]
}
};

var cur=”single”, idx=0;

function esc(t){return t}
function render(){
var d=DATA[cur];
var stream=$(‘#sc-stream’); stream.innerHTML=””;
var vars=[];
for(var i=0;i<=idx && i<d.steps.length;i++){
var s=d.steps[i];
if(s.addVars)vars=vars.concat(s.addVars);
var el=document.createElement(‘div’); el.className=’step’;
var tcls=’think’+(s.revise?’ revise’:”);
var badge=s.revise?’REVISE’:(s.final?’SUBMIT’:’STEP ‘+(i+1));
el.innerHTML='<div class=”‘+tcls+'”><span class=”badge”>’+badge+'</span>’+s.think+'</div>’+
‘<pre>’+s.code+'</pre>’+
‘<div class=”fb”>’+s.fb+'</div>’;
stream.appendChild(el);
}
// state panel
$(‘#sc-statelbl’).textContent=d.label;
var vb=$(‘#sc-vars’);
if(cur===’single’){
vb.innerHTML='<div class=”empty”>’+d.stateNote+'</div>’;
}else if(vars.length===0){
vb.innerHTML='<div class=”empty”>’+d.stateNote+'</div>’;
}else{
vb.innerHTML='<div class=”empty” style=”margin-bottom:9px”>’+d.stateNote+'</div>’+
vars.map(function(v){return ‘<div class=”var”><b>’+v.n+'</b><i>’+v.t+'</i></div>’}).join(”);
}
// verdict
var vdt=$(‘#sc-verdict’);
var last=d.steps[Math.min(idx,d.steps.length-1)];
if(idx>=d.steps.length-1 && last.final){
vdt.className=’verdict show ‘+(last.correct?’good’:’bad’);
vdt.querySelector(‘.mark’).textContent=last.correct?’✓’:’✗’;
$(‘#sc-vtxt’).innerHTML='<b>Submitted answer: ‘+last.answer+(last.correct?’ m’:”)+'</b>’+
‘<small>’+last.why+'</small>’;
}else{ vdt.className=’verdict’; }
// controls
$(‘#sc-prev’).disabled=(idx<=0);
$(‘#sc-next’).disabled=(idx>=d.steps.length-1);
$(‘#sc-next’).textContent=(idx>=d.steps.length-1)?’Done’:’Run next step ‘;
$(‘#sc-prog’).textContent=’step ‘+(idx+1)+’ / ‘+d.steps.length;
resize();
}

function setTab(k){
cur=k; idx=0;
root.querySelectorAll(‘.tab’).forEach(function(t){
t.classList.toggle(‘on’,t.getAttribute(‘data-k’)===k);
});
render();
}

$(‘#sc-tabs’).addEventListener(‘click’,function(e){
var t=e.target.closest(‘.tab’); if(!t)return; setTab(t.getAttribute(‘data-k’));
});
$(‘#sc-next’).addEventListener(‘click’,function(){
if(idx<DATA[cur].steps.length-1){idx++;render();}
});
$(‘#sc-prev’).addEventListener(‘click’,function(){
if(idx>0){idx–;render();}
});
$(‘#sc-reset’).addEventListener(‘click’,function(){idx=0;render();});

// auto-resize for WordPress iframe embedding
function resize(){
try{
var h=root.offsetHeight+40;
if(window.parent && window.parent!==window){
window.parent.postMessage({type:’sc-resize’,height:h},’*’);
}
}catch(e){}
}
window.addEventListener(‘load’,resize);
window.addEventListener(‘resize’,resize);

render();
})();
</script>
“>