Source code: Text-prompted Segment Anything on aerial imagery: Grounding DINO and SAM 2.1 in Python

Runnable Python scripts, requirements and console output for the tutorial “Text-prompted Segment Anything on aerial imagery: Grounding DINO and SAM 2.1…

Last updated

These are the exact scripts used to produce every figure and number in Text-prompted Segment Anything on aerial imagery: Grounding DINO and SAM 2.1 in Python. Download them, install the requirements into a fresh virtual environment and run them from this folder.

Written and tested by

Run it
python -m venv .venv
.venv\Scripts\activate # macOS/Linux: source .venv/bin/activate
pip install -r requirements.txt
python text_prompt_sam.py

text_prompt_sam.py

text_prompt_sam.py
"""Text-prompted Segment Anything on aerial imagery: Grounding DINO + SAM 2.1, scored against the Dutch BGT.
The open equivalent of ArcGIS Pro's "Text SAM": a text prompt such as "building." goes to Grounding DINO, which
returns boxes; each box goes to SAM 2.1, which traces the object's outline. The masks become georeferenced
polygons, which are measured against the Dutch large-scale topographic register (BGT): every building, every
stretch of water and, in Utrecht, every street and park tree the city manages.
Two 512 x 512 m sites of 25 cm summer 2025 aerial photography (PDOK Luchtfoto, CC BY 4.0), both in Utrecht:
* Kanaleneiland: used to choose each class's prompt wording, detection tile size and score threshold;
* Parkwijk, Leidsche Rijn: a newer district 4 km away, used only to test those choices.
Run with: python text_prompt_sam.py
"""
import json
import platform
import time
import warnings
from pathlib import Path
import geopandas as gpd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import rasterio
import rasterio.features
import requests
import shapely
import torch
import transformers
from matplotlib.lines import Line2D
from matplotlib.patches import Patch, Rectangle
from PIL import Image
from pyproj import Transformer
from rasterio.errors import NotGeoreferencedWarning
from rasterio.io import MemoryFile
from rasterio.transform import from_origin
from shapely.geometry import box, shape
from torchvision.ops import nms
from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor, Sam2Model, Sam2Processor
warnings.filterwarnings("ignore", category=NotGeoreferencedWarning)
transformers.logging.set_verbosity_error()
transformers.utils.logging.disable_progress_bar()
HERE = Path(__file__).parent
OUT, DATA = HERE / "output", HERE / "data"
for d in (OUT, DATA):
d.mkdir(exist_ok=True)
plt.rcParams.update({"figure.dpi": 150, "savefig.bbox": "tight", "font.size": 9, "figure.constrained_layout.use": True})
pd.set_option("display.width", 160)
torch.set_num_threads(8)
CRS = "EPSG:28992" # Amersfoort / RD New, the Dutch national grid (metres)
def section(title: str) -> None:
print(f"\n=== {title} ===")
section("versions")
print(f"python {platform.python_version()} | torch {torch.__version__} | transformers {transformers.__version__} | "
f"rasterio {rasterio.__version__} | geopandas {gpd.__version__} | shapely {shapely.__version__}")
print(f"device: CPU ({torch.get_num_threads()} threads), CUDA available: {torch.cuda.is_available()}")
# --- Step 1: two sites of 25 cm aerial photography --------------------------------------------
SITES = {
"kanaleneiland": dict(name="Kanaleneiland", lonlat=(5.0990, 52.0720), role="calibration"),
"parkwijk": dict(name="Parkwijk, Leidsche Rijn", lonlat=(5.0590, 52.0985), role="test"),
}
CAL, TEST = "kanaleneiland", "parkwijk"
RES, PX = 0.25, 2048 # 2048 x 2048 pixels of 25 cm = 512 x 512 m
SIZE_M = RES * PX
WMS = "https://service.pdok.nl/hwh/luchtfotorgb/wms/v1_0"
LAYER = "2025_ortho25" # summer 2025, 25 cm; "Actueel_ortho25" always points at the latest year
to_rd = Transformer.from_crs(4326, CRS, always_xy=True)
section("aerial photography (PDOK Luchtfoto 2025 ortho 25 cm)")
for key, s in SITES.items():
cx, cy = (round(v / 10) * 10 for v in to_rd.transform(*s["lonlat"]))
s["bounds"] = (cx - SIZE_M / 2, cy - SIZE_M / 2, cx + SIZE_M / 2, cy + SIZE_M / 2)
s["transform"] = from_origin(s["bounds"][0], s["bounds"][3], RES, RES)
s["box"] = box(*s["bounds"])
tif = DATA / f"{key}_ortho25_2025.tif"
if not tif.exists():
params = dict(service="WMS", version="1.3.0", request="GetMap", layers=LAYER, styles="", crs=CRS,
bbox=",".join(f"{v:.2f}" for v in s["bounds"]), width=PX, height=PX, format="image/png")
r = requests.get(WMS, params=params, timeout=300)
r.raise_for_status()
with MemoryFile(r.content) as mem, mem.open() as img: # PDOK returns a JPEG whatever you ask for
rgb = img.read()[:3]
with rasterio.open(tif, "w", driver="GTiff", width=PX, height=PX, count=3, dtype="uint8", crs=CRS,
transform=s["transform"], compress="deflate", photometric="RGB") as dst:
dst.write(rgb)
with rasterio.open(tif) as src:
s["rgb"] = np.moveaxis(src.read(), 0, -1)
print(f"{s['name']:<24} {s['role']:<11} centre RD {cx:.0f}, {cy:.0f} | {PX} x {PX} px at {RES} m "
f"= {SIZE_M:.0f} x {SIZE_M:.0f} m")
# --- Step 2: reference data from the BGT --------------------------------------------------------
BGT = "https://api.pdok.nl/lv/bgt/ogc/v1/collections/{}/items"
RD_URI = "http://www.opengis.net/def/crs/EPSG/0/28992"
def bgt(collection: str, bounds, path: Path) -> gpd.GeoDataFrame:
"""Current features of one BGT collection in a box, from PDOK's OGC API Features (paged, cached)."""
if not path.exists():
feats, url = [], BGT.format(collection)
params = {"bbox": ",".join(map(str, bounds)), "bbox-crs": RD_URI, "crs": RD_URI, "limit": 1000, "f": "json"}
while url:
r = requests.get(url, params=params, timeout=300)
r.raise_for_status()
page = r.json()
feats += page["features"]
url, params = next((l["href"] for l in page["links"] if l["rel"] == "next"), None), None
gpd.GeoDataFrame.from_features(feats, crs=CRS).to_file(path, driver="GPKG")
g = gpd.read_file(path)
return g[g["eind_registratie"].isna()] # the API also returns every earlier version of each feature
section("reference: BGT (Basisregistratie Grootschalige Topografie)")
for key, s in SITES.items():
site_box = s["box"]
pand = bgt("pand", s["bounds"], DATA / f"{key}_bgt_pand.gpkg")
water = bgt("waterdeel", s["bounds"], DATA / f"{key}_bgt_waterdeel.gpkg")
trees = bgt("vegetatieobject_punt", s["bounds"], DATA / f"{key}_bgt_bomen.gpkg")
roads = bgt("wegdeel", s["bounds"], DATA / f"{key}_bgt_wegdeel.gpkg")
pand = pand[pand["status"].eq("bestaand")] # existing buildings, not planned ones
water = water[water["relatieve_hoogteligging"].ge(0)] # nothing in culverts or tunnels
roads = roads[roads["relatieve_hoogteligging"].ge(0)]
trees = trees[trees["plus_type"].eq("boom") & trees.within(site_box)]
# clip to the site; keep building parts of at least 5 m² (a sliver along the edge is not a building)
pand = pand.assign(geometry=pand.intersection(site_box)).explode(index_parts=False)
pand = pand[pand.area >= 5].reset_index(drop=True)
# each terraced house is its own BGT "pand"; touching buildings merged together make one roof "block"
blocks = gpd.GeoDataFrame(geometry=list(shapely.get_parts(pand.buffer(0.05).union_all())), crs=CRS)
blocks["geometry"] = blocks.buffer(-0.05)
s["ref"] = dict(
building=pand[["geometry"]], block=blocks,
water=gpd.GeoDataFrame(geometry=[water.intersection(site_box).union_all()], crs=CRS),
tree=trees[["geometry"]].reset_index(drop=True),
road=gpd.GeoDataFrame(geometry=[roads.intersection(site_box).union_all()], crs=CRS))
print(f"{s['name']:<24} buildings (pand) {len(pand):>4} | roof blocks {len(blocks):>4} | "
f"water {s['ref']['water'].area.sum() / 1e4:4.2f} ha | registered trees {len(trees):>4} | "
f"road and parking surface {s['ref']['road'].area.sum() / 1e4:5.2f} ha")
# --- Step 3: the two models ------------------------------------------------------------------------
GD_ID, SAM_ID = "IDEA-Research/grounding-dino-base", "facebook/sam2.1-hiera-large"
section("models")
t0 = time.perf_counter()
gd_proc = AutoProcessor.from_pretrained(GD_ID)
gd = AutoModelForZeroShotObjectDetection.from_pretrained(GD_ID).eval()
sam_proc = Sam2Processor.from_pretrained(SAM_ID)
sam = Sam2Model.from_pretrained(SAM_ID).eval()
n_par = lambda m: sum(p.numel() for p in m.parameters()) / 1e6
print(f"{GD_ID}: {n_par(gd):.0f} M parameters | {SAM_ID}: {n_par(sam):.0f} M parameters "
f"| loaded in {time.perf_counter() - t0:.1f} s (both Apache-2.0)")
# --- Step 4: detect with Grounding DINO ----------------------------------------------------------------
# Grounding DINO resizes every image to 800 pixels on its short side. A 1024-pixel tile is shrunk (a car becomes
# about 14 pixels long); a 512-pixel tile is enlarged (the same car becomes about 28). Both are tried. Tiles overlap
# by half and each keeps only the boxes whose centre falls in its own core, so every object is counted once.
CANDIDATES = {"building": ["building.", "house.", "roof."], "tree": ["tree.", "a tree.", "tree crown."],
"water": ["water."], "car": ["car."]}
DET_TILES = (1024, 512)
FLOOR = 0.10 # keep every box above this; thresholds are chosen afterwards from the scores
MAX_BOX_FRAC = 0.5 # a box covering half a tile is "the whole scene", never one object
def offsets(t: int) -> list[int]:
return list(range(0, PX - t + 1, t // 2))
def core(o: int, t: int) -> tuple[int, int]:
lo = 0 if o == 0 else o + t // 4
hi = PX if o + t == PX else o + t - t // 4
return lo, hi
def detect(img: Image.Image, prompt: str):
inputs = gd_proc(images=img, text=prompt, return_tensors="pt")
with torch.inference_mode():
out = gd(**inputs)
res = gd_proc.post_process_grounded_object_detection(
out, inputs.input_ids, threshold=FLOOR, text_threshold=FLOOR, target_sizes=[img.size[::-1]])[0]
boxes, scores = res["boxes"], res["scores"]
area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
keep = area < MAX_BOX_FRAC * img.size[0] * img.size[1]
boxes, scores = boxes[keep], scores[keep]
keep = nms(boxes, scores, 0.5) # one box per object
return boxes[keep], scores[keep]
timings = []
section("Grounding DINO: 6 prompts x 2 tile sizes (9 tiles of 1024 px, 49 of 512 px per site)")
for key, s in SITES.items():
cache = DATA / f"{key}_detections.csv"
if cache.exists():
s["det"] = pd.read_csv(cache)
print(f"{s['name']}: detections read from {cache.name}")
continue
rows = []
for t in DET_TILES:
for oy in offsets(t):
for ox in offsets(t):
img = Image.fromarray(s["rgb"][oy:oy + t, ox:ox + t])
(cx0, cx1), (cy0, cy1) = core(ox, t), core(oy, t)
for cls, prompts in CANDIDATES.items():
for prompt in prompts:
t0 = time.perf_counter()
boxes, scores = detect(img, prompt)
timings.append(dict(step=f"Grounding DINO, {t}-px tile", n=1, s=time.perf_counter() - t0))
for (x0, y0, x1, y1), sc in zip(boxes.tolist(), scores.tolist()):
gx, gy = (x0 + x1) / 2 + ox, (y0 + y1) / 2 + oy
if cx0 <= gx < cx1 and cy0 <= gy < cy1:
rows.append(dict(cls=cls, prompt=prompt, det_tile=t, score=round(sc, 4),
x0=x0 + ox, y0=y0 + oy, x1=x1 + ox, y1=y1 + oy))
print(f" {s['name']}: {t}-px tiles done, {len(rows)} boxes so far", flush=True)
s["det"] = pd.DataFrame(rows)
s["det"].to_csv(cache, index=False)
# --- Step 5: segment every box with SAM 2.1 ---------------------------------------------------------------
# SAM 2.1 encodes a 1024 x 1024 tile once (the slow part), then turns each box into a mask in milliseconds.
TILE = 1024
def segment(embeddings, boxes: list, batch: int = 64):
masks, quality = [], []
for i in range(0, len(boxes), batch):
inputs = sam_proc(input_boxes=[boxes[i:i + batch]], original_sizes=[[TILE, TILE]], return_tensors="pt")
with torch.inference_mode():
out = sam(image_embeddings=embeddings, input_boxes=inputs["input_boxes"], multimask_output=False)
masks.append(sam_proc.post_process_masks(out.pred_masks, [[TILE, TILE]])[0][:, 0].numpy().astype(bool))
quality.append(out.iou_scores[0, :, 0].numpy())
return np.concatenate(masks), np.concatenate(quality)
def to_polygon(mask: np.ndarray, transform, min_part_m2: float = 0.5):
parts = [shape(g) for g, v in rasterio.features.shapes(mask.astype("uint8"), mask=mask, transform=transform)
if v == 1]
parts = [p for p in parts if p.area >= min_part_m2]
return shapely.MultiPolygon(parts).simplify(RES / 2) if parts else None
section("SAM 2.1: one box prompt per detection (9 tiles of 1024 px per site)")
results = {}
for key, s in SITES.items():
cache = DATA / f"{key}_masks.gpkg"
if cache.exists():
results[key] = gpd.read_file(cache)
print(f"{s['name']}: masks read from {cache.name}")
continue
det, rows = s["det"], []
gx, gy = (det.x0 + det.x1) / 2, (det.y0 + det.y1) / 2
for oy in offsets(TILE):
for ox in offsets(TILE):
(cx0, cx1), (cy0, cy1) = core(ox, TILE), core(oy, TILE)
mine = det[(gx >= cx0) & (gx < cx1) & (gy >= cy0) & (gy < cy1)]
tile = Image.fromarray(s["rgb"][oy:oy + TILE, ox:ox + TILE])
t_tf = from_origin(s["bounds"][0] + ox * RES, s["bounds"][3] - oy * RES, RES, RES)
t0 = time.perf_counter()
with torch.inference_mode():
emb = sam.get_image_embeddings(sam_proc(images=tile, return_tensors="pt")["pixel_values"])
timings.append(dict(step="SAM 2.1 image encoder, 1024-px tile", n=1, s=time.perf_counter() - t0))
local = np.clip(mine[["x0", "y0", "x1", "y1"]].to_numpy() - [ox, oy, ox, oy], 0, TILE).tolist()
t0 = time.perf_counter()
masks, quality = segment(emb, local)
timings.append(dict(step="SAM 2.1 mask decoder, per box", n=len(local), s=time.perf_counter() - t0))
for (_, d), m, q in zip(mine.iterrows(), masks, quality):
poly = to_polygon(m, t_tf)
if poly is not None:
rows.append(dict(cls=d.cls, prompt=d.prompt, det_tile=int(d.det_tile), score=d.score,
sam_iou=round(float(q), 4), geometry=poly,
box=box(*(s["transform"] * (d.x0, d.y1)), *(s["transform"] * (d.x1, d.y0))).wkt))
print(f" {s['name']}: row {oy} done, {len(rows)} polygons so far", flush=True)
results[key] = gpd.GeoDataFrame(rows, crs=CRS)
results[key].to_file(cache, driver="GPKG")
timing_csv = DATA / "timings.csv"
if timings: # keep the timings with the caches, so a rerun from cache still reports them
tm = pd.DataFrame(timings).groupby("step")[["n", "s"]].sum()
tm["seconds_each"] = tm.s / tm.n
tm.round(3).rename(columns={"n": "count", "s": "total_s"}).to_csv(timing_csv)
if timing_csv.exists():
if not timings:
print("timings from the run that built the caches in data/:")
print(pd.read_csv(timing_csv, index_col="step").to_string())
counts = pd.concat({SITES[k]["name"]: r.groupby(["cls", "prompt", "det_tile"]).size() for k, r in results.items()},
axis=1).fillna(0).astype(int)
print(f"\npolygons with a box score of at least {FLOOR}:")
print(counts.to_string())
# --- Step 6: scoring ----------------------------------------------------------------------------------
def match_objects(pred: gpd.GeoDataFrame, ref: gpd.GeoDataFrame, thr: float = 0.5):
"""Greedy one-to-one matching by IoU: each reference object can be matched by one prediction at most."""
pred, ref = pred.reset_index(drop=True), ref.reset_index(drop=True)
used_p, used_r = np.zeros(len(pred), bool), np.zeros(len(ref), bool)
if len(pred) == 0 or len(ref) == 0:
return np.array([]), used_p, used_r
pairs = gpd.sjoin(pred[["geometry"]], ref[["geometry"]], predicate="intersects")
pi, ri = pairs.index.values, pairs["index_right"].values
pg, rg = pred.geometry.values[pi], ref.geometry.values[ri]
inter = shapely.area(shapely.intersection(pg, rg))
iou = inter / (shapely.area(pg) + shapely.area(rg) - inter)
ious = []
for k in np.argsort(-iou):
if iou[k] < thr:
break
if not used_p[pi[k]] and not used_r[ri[k]]:
used_p[pi[k]] = used_r[ri[k]] = True
ious.append(iou[k])
return np.array(ious), used_p, used_r
def prf(tp: int, n_pred: int, n_ref: int) -> dict:
p = tp / n_pred if n_pred else 0.0
r = tp / n_ref if n_ref else 0.0
return dict(precision=p, recall=r, f1=2 * p * r / (p + r) if p + r else 0.0)
def raster(geoms, s) -> np.ndarray:
geoms = [g for g in geoms if g is not None and not g.is_empty]
if not geoms:
return np.zeros((PX, PX), bool)
return rasterio.features.rasterize(geoms, out_shape=(PX, PX), transform=s["transform"]).astype(bool)
def pixel_scores(pred_geoms, ref_geoms, s) -> dict:
p, r = raster(pred_geoms, s), raster(ref_geoms, s)
tp, fp, fn = int((p & r).sum()), int((p & ~r).sum()), int((~p & r).sum())
return dict(prf(tp, tp + fp, tp + fn), iou=tp / (tp + fp + fn) if tp + fp + fn else 0.0)
def clip(pred: gpd.GeoDataFrame, s) -> gpd.GeoDataFrame:
pred = pred.assign(geometry=pred.intersection(s["box"]))
return pred[~pred.is_empty].reset_index(drop=True)
def score(cls: str, pred: gpd.GeoDataFrame, s) -> dict:
ref, pred = s["ref"], clip(pred, s)
if cls == "building": # headline: building area right, pixel by pixel; then object by object
_, used_p, _ = match_objects(pred, ref["building"])
_, used_pb, _ = match_objects(pred, ref["block"])
return dict(pixel_scores(pred.geometry, ref["building"].geometry, s), objects=len(pred),
**{f"obj_{k}": v for k, v in prf(int(used_p.sum()), len(pred), len(ref["building"])).items()},
block_f1=prf(int(used_pb.sum()), len(pred), len(ref["block"]))["f1"])
if cls == "tree": # a crown is right if it holds a registered tree; a tree is found if covered
if len(pred) == 0:
return dict(prf(0, 0, len(ref["tree"])), objects=0)
hit = gpd.sjoin(ref["tree"], pred[["geometry"]], predicate="within")
p, r = hit["index_right"].nunique() / len(pred), hit.index.nunique() / len(ref["tree"])
return dict(precision=p, recall=r, f1=2 * p * r / (p + r) if p + r else 0.0, objects=len(pred))
if cls == "water":
return dict(pixel_scores(pred.geometry, ref["water"].geometry, s), objects=len(pred))
# there is no register of parked cars, so check where the detections are instead
road = ref["road"].geometry.iloc[0].buffer(1.0)
return dict(objects=len(pred), on_road=float(pred.representative_point().within(road).mean()) if len(pred)
else 0.0)
THRESHOLDS = np.round(np.arange(0.10, 0.51, 0.025), 3)
HEADLINE = ["building", "tree", "water"] # building and water pixel F1, tree crown F1
section("calibration: every prompt wording, tile size and score threshold")
sweep = []
for key, s in SITES.items():
r = results[key]
for (cls, prompt, t), g in r.groupby(["cls", "prompt", "det_tile"]):
for thr in THRESHOLDS:
sweep.append(dict(site=key, cls=cls, prompt=prompt, det_tile=t, thr=thr,
**score(cls, g[g.score >= thr], s)))
sweep = pd.DataFrame(sweep)
cal = sweep[(sweep.site == CAL) & sweep.cls.isin(HEADLINE)]
best_each = cal.loc[cal.groupby(["cls", "prompt", "det_tile"]).f1.idxmax()]
print("best F1 on Kanaleneiland for each wording and tile size:")
print(best_each[["cls", "prompt", "det_tile", "thr", "objects", "precision", "recall", "f1"]].round(2).to_string(
index=False))
chosen = {c: best_each[best_each.cls == c].sort_values("f1").iloc[-1][["prompt", "det_tile", "thr"]].to_dict()
for c in HEADLINE}
cars = sweep[(sweep.site == CAL) & (sweep.cls == "car")]
print("\ncars on Kanaleneiland (how many, and the share on a BGT road or parking surface):")
print(cars.pivot_table(index="thr", columns="det_tile", values=["objects", "on_road"]).round(2).iloc[::2].to_string())
ok = cars[cars.on_road >= 0.95].sort_values(["objects", "thr"], ascending=[False, True])
chosen["car"] = ok.iloc[0][["prompt", "det_tile", "thr"]].to_dict()
for c, v in chosen.items():
v["det_tile"], v["thr"] = int(v["det_tile"]), round(float(v["thr"]), 3)
print("\nchosen: " + "; ".join(f"{c}: '{v['prompt']}' on {v['det_tile']}-px tiles, score >= {v['thr']}"
for c, v in chosen.items()))
print("(building, tree, water: highest F1; car: the most cars with at least 95% on a road or parking surface)")
# --- Step 7: results on both sites with the chosen settings ------------------------------------------------
section("accuracy with the Kanaleneiland settings")
final, metrics = {}, {}
for key, s in SITES.items():
r = results[key]
final[key] = {c: r[(r.cls == c) & (r.prompt == v["prompt"]) & (r.det_tile == v["det_tile"])
& (r.score >= v["thr"])].reset_index(drop=True) for c, v in chosen.items()}
metrics[key] = {c: score(c, final[key][c], s) for c in chosen}
m = metrics[key]
print(f"\n{s['name']} ({s['role']})")
b = m["building"]
print(f" building {b['objects']:>5} polygons | pixels: precision {b['precision']:.0%}, recall "
f"{b['recall']:.0%}, IoU {b['iou']:.2f}, F1 {b['f1']:.2f} | objects (IoU >= 0.5): F1 "
f"{b['obj_f1']:.2f} against single BGT pand, {b['block_f1']:.2f} against roof blocks")
t = m["tree"]
print(f" tree {t['objects']:>5} crowns | {t['precision']:.0%} hold a registered tree; "
f"{t['recall']:.0%} of {len(s['ref']['tree'])} registered trees are inside a crown; F1 {t['f1']:.2f}")
w = m["water"]
print(f" water {w['objects']:>5} polygons | pixels: precision {w['precision']:.0%}, recall "
f"{w['recall']:.0%}, IoU {w['iou']:.2f}, F1 {w['f1']:.2f}")
c = m["car"]
print(f" car {c['objects']:>5} cars | {c['on_road']:.0%} on a BGT road or parking surface")
own = sweep[(sweep.site == key) & sweep.cls.isin(HEADLINE)].groupby("cls").f1.max()
print(" best F1 this site could have reached with its own settings: "
+ ", ".join(f"{c} {own[c]:.2f}" for c in HEADLINE))
section("what SAM adds: the Grounding DINO boxes alone as building outlines")
for key, s in SITES.items():
sel = final[key]["building"]
sb = score("building", sel.assign(geometry=gpd.GeoSeries.from_wkt(sel["box"], crs=CRS)), s)
metrics[key]["building_boxes_only"] = sb
sm = metrics[key]["building"]
print(f"{s['name']:<24} boxes: pixel IoU {sb['iou']:.2f}, F1 {sb['f1']:.2f}, object F1 {sb['obj_f1']:.2f} | "
f"SAM 2.1 masks: pixel IoU {sm['iou']:.2f}, F1 {sm['f1']:.2f}, object F1 {sm['obj_f1']:.2f}")
section("Parkwijk: share of BGT buildings found, by footprint size")
s = SITES[TEST]
ious, _, used_r = match_objects(clip(final[TEST]["building"], s), s["ref"]["building"])
ref_b = s["ref"]["building"].assign(found=used_r, area=s["ref"]["building"].area)
ref_b["size"] = pd.cut(ref_b.area, [0, 20, 50, 150, 500, 1e9],
labels=["under 20 m²", "20-50 m²", "50-150 m²", "150-500 m²", "over 500 m²"])
print(ref_b.groupby("size", observed=True).agg(buildings=("found", "size"), recall=("found", "mean")).round(2)
.to_string())
print(f"median IoU of the matched buildings: {np.median(ious):.2f}")
# --- Step 8: GeoPackage and metrics ------------------------------------------------------------------------
section("outputs")
gpkg = OUT / "text_prompt_sam.gpkg"
gpkg.unlink(missing_ok=True)
for key in SITES:
for cls, g in final[key].items():
if len(g):
g.drop(columns="box").to_file(gpkg, layer=f"{key}_{cls}", driver="GPKG")
print(f"wrote {sum(len(g) for f in final.values() for g in f.values())} polygons in "
f"{len(SITES) * len(chosen)} layers to {gpkg.name} (EPSG:28992)")
(OUT / "metrics.json").write_text(json.dumps(dict(chosen=chosen, metrics=metrics), indent=1, default=float))
# --- Step 9: figures -------------------------------------------------------------------------------------
COL = {"building": "#e8590c", "tree": "#2b8a3e", "water": "#1c7ed6", "car": "#f2c94c"}
def show(ax, s, window=None):
ax.imshow(s["rgb"], extent=(s["bounds"][0], s["bounds"][2], s["bounds"][1], s["bounds"][3]))
if window:
ax.set_xlim(window[0], window[2])
ax.set_ylim(window[1], window[3])
ax.set_xticks([])
ax.set_yticks([])
def save(fig, name: str) -> None:
for ax in fig.axes: # GeoPandas labels map axes "Easting"/"Northing"; the scale bar says enough
if not ax.get_xticks().size:
ax.set_xlabel("")
ax.set_ylabel("")
fig.savefig(OUT / name)
plt.close(fig)
def scalebar(ax, metres, s, window=None):
x0, y0, x1, _ = window or s["bounds"]
w = x1 - x0
ax.plot([x0 + 0.04 * w, x0 + 0.04 * w + metres], [y0 + 0.04 * w] * 2, color="white", lw=3)
ax.text(x0 + 0.04 * w + metres / 2, y0 + 0.06 * w, f"{metres} m", color="white", ha="center", fontsize=8,
fontweight="bold")
# 1. the two sites with their BGT reference
fig, axes = plt.subplots(1, 2, figsize=(11, 5.8))
for ax, (key, s) in zip(axes, SITES.items()):
show(ax, s)
s["ref"]["building"].boundary.plot(ax=ax, color=COL["building"], lw=0.5)
s["ref"]["water"].boundary.plot(ax=ax, color="#4dabf7", lw=1.2)
s["ref"]["tree"].plot(ax=ax, color="#69db7c", markersize=1.5)
scalebar(ax, 100, s)
ax.set_title(f"{s['name']} ({s['role']}): {len(s['ref']['building'])} buildings, "
f"{len(s['ref']['tree'])} registered trees", fontsize=9)
fig.legend(handles=[Line2D([], [], color=COL["building"], label="BGT building (pand)"),
Line2D([], [], color="#4dabf7", label="BGT water"),
Line2D([], [], color="#69db7c", marker="o", ls="", ms=3, label="BGT tree")],
loc="lower center", ncol=3, frameon=False)
fig.suptitle("Two 512 m sites of 25 cm aerial photography (PDOK Luchtfoto 2025) and the BGT reference")
save(fig, "01-sites-bgt-reference.png")
# 2. the pipeline on one window: prompt -> boxes -> masks -> polygons
s = SITES[CAL]
cx, cy = (s["bounds"][0] + s["bounds"][2]) / 2, (s["bounds"][1] + s["bounds"][3]) / 2
win = (cx - 250, cy - 250, cx - 90, cy - 90)
wbox = box(*win)
v = chosen["building"]
r = results[CAL]
b_all = r[(r.cls == "building") & (r.prompt == v["prompt"]) & (r.det_tile == v["det_tile"])]
b_sel = final[CAL]["building"]
fig, axes = plt.subplots(1, 3, figsize=(12, 4.4))
show(axes[0], s, win)
for _, row in b_all[b_all.intersects(wbox)].iterrows():
bx = shapely.from_wkt(row["box"]).bounds
kept = row["score"] >= v["thr"]
axes[0].add_patch(Rectangle(bx[:2], bx[2] - bx[0], bx[3] - bx[1], fill=False, lw=1.2 if kept else 0.7,
ec=COL["building"] if kept else "white", ls="-" if kept else ":"))
axes[0].set_title(f"1. Grounding DINO boxes for '{v['prompt']}'\nsolid: score >= {v['thr']}; dotted: below")
show(axes[1], s, win)
b_sel[b_sel.intersects(wbox)].plot(ax=axes[1], color=COL["building"], alpha=0.55, ec="white", lw=0.5)
axes[1].set_title("2. SAM 2.1 traces the object\nin each kept box")
show(axes[2], s, win)
s["ref"]["building"].boundary.plot(ax=axes[2], color="cyan", lw=0.8)
b_sel[b_sel.intersects(wbox)].boundary.plot(ax=axes[2], color=COL["building"], lw=1.2)
axes[2].set_title("3. Polygons (orange) against\nBGT pand (cyan)")
for ax in axes:
scalebar(ax, 25, s, win)
save(fig, "02-pipeline-boxes-masks-polygons.png")
# 3. all four prompts on each site
fig, axes = plt.subplots(1, 2, figsize=(11, 5.8))
for ax, (key, s) in zip(axes, SITES.items()):
show(ax, s)
for cls in ["water", "tree", "building", "car"]:
if len(final[key][cls]):
final[key][cls].plot(ax=ax, color=COL[cls], alpha=0.6, lw=0)
scalebar(ax, 100, s)
ax.set_title(s["name"], fontsize=9)
fig.legend(handles=[Patch(color=COL[c], alpha=0.7, label=f"'{v['prompt']}' ({v['det_tile']} px, >= {v['thr']})")
for c, v in chosen.items()], loc="lower center", ncol=4, frameon=False)
fig.suptitle("Four text prompts with the settings chosen on Kanaleneiland")
save(fig, "03-four-prompts-both-sites.png")
# 4. wording and tile size
fig, axes = plt.subplots(1, 2, figsize=(11, 3.4), gridspec_kw={"width_ratios": [3, 3]})
for ax, cls in zip(axes, ["building", "tree"]):
d = best_each[best_each.cls == cls].pivot(index="prompt", columns="det_tile", values="f1")
d = d.loc[CANDIDATES[cls]]
d.plot.barh(ax=ax, color={1024: "#adb5bd", 512: "#495057"}, width=0.7)
ax.set_xlabel("best F1 on Kanaleneiland")
ax.set_ylabel("")
ax.set_title(f"{cls}: {'pixel F1' if cls == 'building' else 'crown F1'}")
ax.legend(title="tile (px)", fontsize=8, loc="lower right")
ax.invert_yaxis()
fig.suptitle("The same objects, worded differently and cut into different tile sizes")
save(fig, "04-prompt-wording-tile-size.png")
# 5. threshold sweep for the chosen settings
TITLES = {"building": "building: pixel F1", "tree": "tree: crown F1",
"water": "water: pixel F1"}
fig, axes = plt.subplots(1, 3, figsize=(12, 3.6), sharey=True)
for ax, cls in zip(axes, HEADLINE):
v = chosen[cls]
for key, s in SITES.items():
d = sweep[(sweep.site == key) & (sweep.cls == cls) & (sweep.prompt == v["prompt"])
& (sweep.det_tile == v["det_tile"])]
ax.plot(d.thr, d.f1, marker="o", ms=3, label=s["name"], ls="-" if key == CAL else "--")
ax.axvline(v["thr"], color="grey", lw=0.8)
ax.set_title(f"{TITLES[cls]}\n'{v['prompt']}', {v['det_tile']}-px tiles")
ax.set_xlabel("Grounding DINO box score threshold")
ax.grid(alpha=0.3)
axes[0].set_ylabel("F1")
axes[0].legend(fontsize=8)
fig.suptitle("Choose the threshold on one site (grey line), then check it on another")
save(fig, "05-threshold-sweep.png")
# 6. building errors on the test site
s = SITES[TEST]
pred_b = clip(final[TEST]["building"], s)
_, used_p, used_r = match_objects(pred_b, s["ref"]["building"])
fig, ax = plt.subplots(figsize=(7.5, 7.5))
show(ax, s)
s["ref"]["building"][~used_r].plot(ax=ax, color="none", ec="#e03131", lw=0.9)
if used_p.any():
pred_b[used_p].plot(ax=ax, color="#40c057", alpha=0.5, lw=0)
if (~used_p).any():
pred_b[~used_p].plot(ax=ax, color="#fab005", alpha=0.6, lw=0)
scalebar(ax, 100, s)
ax.legend(handles=[Patch(color="#40c057", alpha=0.6, label=f"matches one BGT building (IoU >= 0.5): {used_p.sum()}"),
Patch(color="#fab005", alpha=0.7, label=f"polygon with no match: {(~used_p).sum()}"),
Patch(fc="none", ec="#e03131", label=f"BGT building not matched: {(~used_r).sum()}")],
loc="upper left", fontsize=8, framealpha=0.85)
mb = metrics[TEST]["building"]
ax.set_title(f"{s['name']}, '{chosen['building']['prompt']}': pixel F1 {mb['f1']:.2f}, "
f"object F1 {mb['obj_f1']:.2f}")
save(fig, "06-parkwijk-building-errors.png")
print("figures written to output/")