"""Text-prompted Segment Anything on aerial imagery: Grounding DINO + SAM 2.1, scored against the Dutch BGT. The open equivalent of ArcGIS Pro's "Text SAM": a text prompt such as "building." goes to Grounding DINO, which returns boxes; each box goes to SAM 2.1, which traces the object's outline. The masks become georeferenced polygons, which are measured against the Dutch large-scale topographic register (BGT): every building, every stretch of water and, in Utrecht, every street and park tree the city manages. Two 512 x 512 m sites of 25 cm summer 2025 aerial photography (PDOK Luchtfoto, CC BY 4.0), both in Utrecht: * Kanaleneiland: used to choose each class's prompt wording, detection tile size and score threshold; * Parkwijk, Leidsche Rijn: a newer district 4 km away, used only to test those choices. Run with: python text_prompt_sam.py """ import json import platform import time import warnings from pathlib import Path import geopandas as gpd import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pandas as pd import rasterio import rasterio.features import requests import shapely import torch import transformers from matplotlib.lines import Line2D from matplotlib.patches import Patch, Rectangle from PIL import Image from pyproj import Transformer from rasterio.errors import NotGeoreferencedWarning from rasterio.io import MemoryFile from rasterio.transform import from_origin from shapely.geometry import box, shape from torchvision.ops import nms from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor, Sam2Model, Sam2Processor warnings.filterwarnings("ignore", category=NotGeoreferencedWarning) transformers.logging.set_verbosity_error() transformers.utils.logging.disable_progress_bar() HERE = Path(__file__).parent OUT, DATA = HERE / "output", HERE / "data" for d in (OUT, DATA): d.mkdir(exist_ok=True) plt.rcParams.update({"figure.dpi": 150, "savefig.bbox": "tight", "font.size": 9, "figure.constrained_layout.use": True}) pd.set_option("display.width", 160) torch.set_num_threads(8) CRS = "EPSG:28992" # Amersfoort / RD New, the Dutch national grid (metres) def section(title: str) -> None: print(f"\n=== {title} ===") section("versions") print(f"python {platform.python_version()} | torch {torch.__version__} | transformers {transformers.__version__} | " f"rasterio {rasterio.__version__} | geopandas {gpd.__version__} | shapely {shapely.__version__}") print(f"device: CPU ({torch.get_num_threads()} threads), CUDA available: {torch.cuda.is_available()}") # --- Step 1: two sites of 25 cm aerial photography -------------------------------------------- SITES = { "kanaleneiland": dict(name="Kanaleneiland", lonlat=(5.0990, 52.0720), role="calibration"), "parkwijk": dict(name="Parkwijk, Leidsche Rijn", lonlat=(5.0590, 52.0985), role="test"), } CAL, TEST = "kanaleneiland", "parkwijk" RES, PX = 0.25, 2048 # 2048 x 2048 pixels of 25 cm = 512 x 512 m SIZE_M = RES * PX WMS = "https://service.pdok.nl/hwh/luchtfotorgb/wms/v1_0" LAYER = "2025_ortho25" # summer 2025, 25 cm; "Actueel_ortho25" always points at the latest year to_rd = Transformer.from_crs(4326, CRS, always_xy=True) section("aerial photography (PDOK Luchtfoto 2025 ortho 25 cm)") for key, s in SITES.items(): cx, cy = (round(v / 10) * 10 for v in to_rd.transform(*s["lonlat"])) s["bounds"] = (cx - SIZE_M / 2, cy - SIZE_M / 2, cx + SIZE_M / 2, cy + SIZE_M / 2) s["transform"] = from_origin(s["bounds"][0], s["bounds"][3], RES, RES) s["box"] = box(*s["bounds"]) tif = DATA / f"{key}_ortho25_2025.tif" if not tif.exists(): params = dict(service="WMS", version="1.3.0", request="GetMap", layers=LAYER, styles="", crs=CRS, bbox=",".join(f"{v:.2f}" for v in s["bounds"]), width=PX, height=PX, format="image/png") r = requests.get(WMS, params=params, timeout=300) r.raise_for_status() with MemoryFile(r.content) as mem, mem.open() as img: # PDOK returns a JPEG whatever you ask for rgb = img.read()[:3] with rasterio.open(tif, "w", driver="GTiff", width=PX, height=PX, count=3, dtype="uint8", crs=CRS, transform=s["transform"], compress="deflate", photometric="RGB") as dst: dst.write(rgb) with rasterio.open(tif) as src: s["rgb"] = np.moveaxis(src.read(), 0, -1) print(f"{s['name']:<24} {s['role']:<11} centre RD {cx:.0f}, {cy:.0f} | {PX} x {PX} px at {RES} m " f"= {SIZE_M:.0f} x {SIZE_M:.0f} m") # --- Step 2: reference data from the BGT -------------------------------------------------------- BGT = "https://api.pdok.nl/lv/bgt/ogc/v1/collections/{}/items" RD_URI = "http://www.opengis.net/def/crs/EPSG/0/28992" def bgt(collection: str, bounds, path: Path) -> gpd.GeoDataFrame: """Current features of one BGT collection in a box, from PDOK's OGC API Features (paged, cached).""" if not path.exists(): feats, url = [], BGT.format(collection) params = {"bbox": ",".join(map(str, bounds)), "bbox-crs": RD_URI, "crs": RD_URI, "limit": 1000, "f": "json"} while url: r = requests.get(url, params=params, timeout=300) r.raise_for_status() page = r.json() feats += page["features"] url, params = next((l["href"] for l in page["links"] if l["rel"] == "next"), None), None gpd.GeoDataFrame.from_features(feats, crs=CRS).to_file(path, driver="GPKG") g = gpd.read_file(path) return g[g["eind_registratie"].isna()] # the API also returns every earlier version of each feature section("reference: BGT (Basisregistratie Grootschalige Topografie)") for key, s in SITES.items(): site_box = s["box"] pand = bgt("pand", s["bounds"], DATA / f"{key}_bgt_pand.gpkg") water = bgt("waterdeel", s["bounds"], DATA / f"{key}_bgt_waterdeel.gpkg") trees = bgt("vegetatieobject_punt", s["bounds"], DATA / f"{key}_bgt_bomen.gpkg") roads = bgt("wegdeel", s["bounds"], DATA / f"{key}_bgt_wegdeel.gpkg") pand = pand[pand["status"].eq("bestaand")] # existing buildings, not planned ones water = water[water["relatieve_hoogteligging"].ge(0)] # nothing in culverts or tunnels roads = roads[roads["relatieve_hoogteligging"].ge(0)] trees = trees[trees["plus_type"].eq("boom") & trees.within(site_box)] # clip to the site; keep building parts of at least 5 m² (a sliver along the edge is not a building) pand = pand.assign(geometry=pand.intersection(site_box)).explode(index_parts=False) pand = pand[pand.area >= 5].reset_index(drop=True) # each terraced house is its own BGT "pand"; touching buildings merged together make one roof "block" blocks = gpd.GeoDataFrame(geometry=list(shapely.get_parts(pand.buffer(0.05).union_all())), crs=CRS) blocks["geometry"] = blocks.buffer(-0.05) s["ref"] = dict( building=pand[["geometry"]], block=blocks, water=gpd.GeoDataFrame(geometry=[water.intersection(site_box).union_all()], crs=CRS), tree=trees[["geometry"]].reset_index(drop=True), road=gpd.GeoDataFrame(geometry=[roads.intersection(site_box).union_all()], crs=CRS)) print(f"{s['name']:<24} buildings (pand) {len(pand):>4} | roof blocks {len(blocks):>4} | " f"water {s['ref']['water'].area.sum() / 1e4:4.2f} ha | registered trees {len(trees):>4} | " f"road and parking surface {s['ref']['road'].area.sum() / 1e4:5.2f} ha") # --- Step 3: the two models ------------------------------------------------------------------------ GD_ID, SAM_ID = "IDEA-Research/grounding-dino-base", "facebook/sam2.1-hiera-large" section("models") t0 = time.perf_counter() gd_proc = AutoProcessor.from_pretrained(GD_ID) gd = AutoModelForZeroShotObjectDetection.from_pretrained(GD_ID).eval() sam_proc = Sam2Processor.from_pretrained(SAM_ID) sam = Sam2Model.from_pretrained(SAM_ID).eval() n_par = lambda m: sum(p.numel() for p in m.parameters()) / 1e6 print(f"{GD_ID}: {n_par(gd):.0f} M parameters | {SAM_ID}: {n_par(sam):.0f} M parameters " f"| loaded in {time.perf_counter() - t0:.1f} s (both Apache-2.0)") # --- Step 4: detect with Grounding DINO ---------------------------------------------------------------- # Grounding DINO resizes every image to 800 pixels on its short side. A 1024-pixel tile is shrunk (a car becomes # about 14 pixels long); a 512-pixel tile is enlarged (the same car becomes about 28). Both are tried. Tiles overlap # by half and each keeps only the boxes whose centre falls in its own core, so every object is counted once. CANDIDATES = {"building": ["building.", "house.", "roof."], "tree": ["tree.", "a tree.", "tree crown."], "water": ["water."], "car": ["car."]} DET_TILES = (1024, 512) FLOOR = 0.10 # keep every box above this; thresholds are chosen afterwards from the scores MAX_BOX_FRAC = 0.5 # a box covering half a tile is "the whole scene", never one object def offsets(t: int) -> list[int]: return list(range(0, PX - t + 1, t // 2)) def core(o: int, t: int) -> tuple[int, int]: lo = 0 if o == 0 else o + t // 4 hi = PX if o + t == PX else o + t - t // 4 return lo, hi def detect(img: Image.Image, prompt: str): inputs = gd_proc(images=img, text=prompt, return_tensors="pt") with torch.inference_mode(): out = gd(**inputs) res = gd_proc.post_process_grounded_object_detection( out, inputs.input_ids, threshold=FLOOR, text_threshold=FLOOR, target_sizes=[img.size[::-1]])[0] boxes, scores = res["boxes"], res["scores"] area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) keep = area < MAX_BOX_FRAC * img.size[0] * img.size[1] boxes, scores = boxes[keep], scores[keep] keep = nms(boxes, scores, 0.5) # one box per object return boxes[keep], scores[keep] timings = [] section("Grounding DINO: 6 prompts x 2 tile sizes (9 tiles of 1024 px, 49 of 512 px per site)") for key, s in SITES.items(): cache = DATA / f"{key}_detections.csv" if cache.exists(): s["det"] = pd.read_csv(cache) print(f"{s['name']}: detections read from {cache.name}") continue rows = [] for t in DET_TILES: for oy in offsets(t): for ox in offsets(t): img = Image.fromarray(s["rgb"][oy:oy + t, ox:ox + t]) (cx0, cx1), (cy0, cy1) = core(ox, t), core(oy, t) for cls, prompts in CANDIDATES.items(): for prompt in prompts: t0 = time.perf_counter() boxes, scores = detect(img, prompt) timings.append(dict(step=f"Grounding DINO, {t}-px tile", n=1, s=time.perf_counter() - t0)) for (x0, y0, x1, y1), sc in zip(boxes.tolist(), scores.tolist()): gx, gy = (x0 + x1) / 2 + ox, (y0 + y1) / 2 + oy if cx0 <= gx < cx1 and cy0 <= gy < cy1: rows.append(dict(cls=cls, prompt=prompt, det_tile=t, score=round(sc, 4), x0=x0 + ox, y0=y0 + oy, x1=x1 + ox, y1=y1 + oy)) print(f" {s['name']}: {t}-px tiles done, {len(rows)} boxes so far", flush=True) s["det"] = pd.DataFrame(rows) s["det"].to_csv(cache, index=False) # --- Step 5: segment every box with SAM 2.1 --------------------------------------------------------------- # SAM 2.1 encodes a 1024 x 1024 tile once (the slow part), then turns each box into a mask in milliseconds. TILE = 1024 def segment(embeddings, boxes: list, batch: int = 64): masks, quality = [], [] for i in range(0, len(boxes), batch): inputs = sam_proc(input_boxes=[boxes[i:i + batch]], original_sizes=[[TILE, TILE]], return_tensors="pt") with torch.inference_mode(): out = sam(image_embeddings=embeddings, input_boxes=inputs["input_boxes"], multimask_output=False) masks.append(sam_proc.post_process_masks(out.pred_masks, [[TILE, TILE]])[0][:, 0].numpy().astype(bool)) quality.append(out.iou_scores[0, :, 0].numpy()) return np.concatenate(masks), np.concatenate(quality) def to_polygon(mask: np.ndarray, transform, min_part_m2: float = 0.5): parts = [shape(g) for g, v in rasterio.features.shapes(mask.astype("uint8"), mask=mask, transform=transform) if v == 1] parts = [p for p in parts if p.area >= min_part_m2] return shapely.MultiPolygon(parts).simplify(RES / 2) if parts else None section("SAM 2.1: one box prompt per detection (9 tiles of 1024 px per site)") results = {} for key, s in SITES.items(): cache = DATA / f"{key}_masks.gpkg" if cache.exists(): results[key] = gpd.read_file(cache) print(f"{s['name']}: masks read from {cache.name}") continue det, rows = s["det"], [] gx, gy = (det.x0 + det.x1) / 2, (det.y0 + det.y1) / 2 for oy in offsets(TILE): for ox in offsets(TILE): (cx0, cx1), (cy0, cy1) = core(ox, TILE), core(oy, TILE) mine = det[(gx >= cx0) & (gx < cx1) & (gy >= cy0) & (gy < cy1)] tile = Image.fromarray(s["rgb"][oy:oy + TILE, ox:ox + TILE]) t_tf = from_origin(s["bounds"][0] + ox * RES, s["bounds"][3] - oy * RES, RES, RES) t0 = time.perf_counter() with torch.inference_mode(): emb = sam.get_image_embeddings(sam_proc(images=tile, return_tensors="pt")["pixel_values"]) timings.append(dict(step="SAM 2.1 image encoder, 1024-px tile", n=1, s=time.perf_counter() - t0)) local = np.clip(mine[["x0", "y0", "x1", "y1"]].to_numpy() - [ox, oy, ox, oy], 0, TILE).tolist() t0 = time.perf_counter() masks, quality = segment(emb, local) timings.append(dict(step="SAM 2.1 mask decoder, per box", n=len(local), s=time.perf_counter() - t0)) for (_, d), m, q in zip(mine.iterrows(), masks, quality): poly = to_polygon(m, t_tf) if poly is not None: rows.append(dict(cls=d.cls, prompt=d.prompt, det_tile=int(d.det_tile), score=d.score, sam_iou=round(float(q), 4), geometry=poly, box=box(*(s["transform"] * (d.x0, d.y1)), *(s["transform"] * (d.x1, d.y0))).wkt)) print(f" {s['name']}: row {oy} done, {len(rows)} polygons so far", flush=True) results[key] = gpd.GeoDataFrame(rows, crs=CRS) results[key].to_file(cache, driver="GPKG") timing_csv = DATA / "timings.csv" if timings: # keep the timings with the caches, so a rerun from cache still reports them tm = pd.DataFrame(timings).groupby("step")[["n", "s"]].sum() tm["seconds_each"] = tm.s / tm.n tm.round(3).rename(columns={"n": "count", "s": "total_s"}).to_csv(timing_csv) if timing_csv.exists(): if not timings: print("timings from the run that built the caches in data/:") print(pd.read_csv(timing_csv, index_col="step").to_string()) counts = pd.concat({SITES[k]["name"]: r.groupby(["cls", "prompt", "det_tile"]).size() for k, r in results.items()}, axis=1).fillna(0).astype(int) print(f"\npolygons with a box score of at least {FLOOR}:") print(counts.to_string()) # --- Step 6: scoring ---------------------------------------------------------------------------------- def match_objects(pred: gpd.GeoDataFrame, ref: gpd.GeoDataFrame, thr: float = 0.5): """Greedy one-to-one matching by IoU: each reference object can be matched by one prediction at most.""" pred, ref = pred.reset_index(drop=True), ref.reset_index(drop=True) used_p, used_r = np.zeros(len(pred), bool), np.zeros(len(ref), bool) if len(pred) == 0 or len(ref) == 0: return np.array([]), used_p, used_r pairs = gpd.sjoin(pred[["geometry"]], ref[["geometry"]], predicate="intersects") pi, ri = pairs.index.values, pairs["index_right"].values pg, rg = pred.geometry.values[pi], ref.geometry.values[ri] inter = shapely.area(shapely.intersection(pg, rg)) iou = inter / (shapely.area(pg) + shapely.area(rg) - inter) ious = [] for k in np.argsort(-iou): if iou[k] < thr: break if not used_p[pi[k]] and not used_r[ri[k]]: used_p[pi[k]] = used_r[ri[k]] = True ious.append(iou[k]) return np.array(ious), used_p, used_r def prf(tp: int, n_pred: int, n_ref: int) -> dict: p = tp / n_pred if n_pred else 0.0 r = tp / n_ref if n_ref else 0.0 return dict(precision=p, recall=r, f1=2 * p * r / (p + r) if p + r else 0.0) def raster(geoms, s) -> np.ndarray: geoms = [g for g in geoms if g is not None and not g.is_empty] if not geoms: return np.zeros((PX, PX), bool) return rasterio.features.rasterize(geoms, out_shape=(PX, PX), transform=s["transform"]).astype(bool) def pixel_scores(pred_geoms, ref_geoms, s) -> dict: p, r = raster(pred_geoms, s), raster(ref_geoms, s) tp, fp, fn = int((p & r).sum()), int((p & ~r).sum()), int((~p & r).sum()) return dict(prf(tp, tp + fp, tp + fn), iou=tp / (tp + fp + fn) if tp + fp + fn else 0.0) def clip(pred: gpd.GeoDataFrame, s) -> gpd.GeoDataFrame: pred = pred.assign(geometry=pred.intersection(s["box"])) return pred[~pred.is_empty].reset_index(drop=True) def score(cls: str, pred: gpd.GeoDataFrame, s) -> dict: ref, pred = s["ref"], clip(pred, s) if cls == "building": # headline: building area right, pixel by pixel; then object by object _, used_p, _ = match_objects(pred, ref["building"]) _, used_pb, _ = match_objects(pred, ref["block"]) return dict(pixel_scores(pred.geometry, ref["building"].geometry, s), objects=len(pred), **{f"obj_{k}": v for k, v in prf(int(used_p.sum()), len(pred), len(ref["building"])).items()}, block_f1=prf(int(used_pb.sum()), len(pred), len(ref["block"]))["f1"]) if cls == "tree": # a crown is right if it holds a registered tree; a tree is found if covered if len(pred) == 0: return dict(prf(0, 0, len(ref["tree"])), objects=0) hit = gpd.sjoin(ref["tree"], pred[["geometry"]], predicate="within") p, r = hit["index_right"].nunique() / len(pred), hit.index.nunique() / len(ref["tree"]) return dict(precision=p, recall=r, f1=2 * p * r / (p + r) if p + r else 0.0, objects=len(pred)) if cls == "water": return dict(pixel_scores(pred.geometry, ref["water"].geometry, s), objects=len(pred)) # there is no register of parked cars, so check where the detections are instead road = ref["road"].geometry.iloc[0].buffer(1.0) return dict(objects=len(pred), on_road=float(pred.representative_point().within(road).mean()) if len(pred) else 0.0) THRESHOLDS = np.round(np.arange(0.10, 0.51, 0.025), 3) HEADLINE = ["building", "tree", "water"] # building and water pixel F1, tree crown F1 section("calibration: every prompt wording, tile size and score threshold") sweep = [] for key, s in SITES.items(): r = results[key] for (cls, prompt, t), g in r.groupby(["cls", "prompt", "det_tile"]): for thr in THRESHOLDS: sweep.append(dict(site=key, cls=cls, prompt=prompt, det_tile=t, thr=thr, **score(cls, g[g.score >= thr], s))) sweep = pd.DataFrame(sweep) cal = sweep[(sweep.site == CAL) & sweep.cls.isin(HEADLINE)] best_each = cal.loc[cal.groupby(["cls", "prompt", "det_tile"]).f1.idxmax()] print("best F1 on Kanaleneiland for each wording and tile size:") print(best_each[["cls", "prompt", "det_tile", "thr", "objects", "precision", "recall", "f1"]].round(2).to_string( index=False)) chosen = {c: best_each[best_each.cls == c].sort_values("f1").iloc[-1][["prompt", "det_tile", "thr"]].to_dict() for c in HEADLINE} cars = sweep[(sweep.site == CAL) & (sweep.cls == "car")] print("\ncars on Kanaleneiland (how many, and the share on a BGT road or parking surface):") print(cars.pivot_table(index="thr", columns="det_tile", values=["objects", "on_road"]).round(2).iloc[::2].to_string()) ok = cars[cars.on_road >= 0.95].sort_values(["objects", "thr"], ascending=[False, True]) chosen["car"] = ok.iloc[0][["prompt", "det_tile", "thr"]].to_dict() for c, v in chosen.items(): v["det_tile"], v["thr"] = int(v["det_tile"]), round(float(v["thr"]), 3) print("\nchosen: " + "; ".join(f"{c}: '{v['prompt']}' on {v['det_tile']}-px tiles, score >= {v['thr']}" for c, v in chosen.items())) print("(building, tree, water: highest F1; car: the most cars with at least 95% on a road or parking surface)") # --- Step 7: results on both sites with the chosen settings ------------------------------------------------ section("accuracy with the Kanaleneiland settings") final, metrics = {}, {} for key, s in SITES.items(): r = results[key] final[key] = {c: r[(r.cls == c) & (r.prompt == v["prompt"]) & (r.det_tile == v["det_tile"]) & (r.score >= v["thr"])].reset_index(drop=True) for c, v in chosen.items()} metrics[key] = {c: score(c, final[key][c], s) for c in chosen} m = metrics[key] print(f"\n{s['name']} ({s['role']})") b = m["building"] print(f" building {b['objects']:>5} polygons | pixels: precision {b['precision']:.0%}, recall " f"{b['recall']:.0%}, IoU {b['iou']:.2f}, F1 {b['f1']:.2f} | objects (IoU >= 0.5): F1 " f"{b['obj_f1']:.2f} against single BGT pand, {b['block_f1']:.2f} against roof blocks") t = m["tree"] print(f" tree {t['objects']:>5} crowns | {t['precision']:.0%} hold a registered tree; " f"{t['recall']:.0%} of {len(s['ref']['tree'])} registered trees are inside a crown; F1 {t['f1']:.2f}") w = m["water"] print(f" water {w['objects']:>5} polygons | pixels: precision {w['precision']:.0%}, recall " f"{w['recall']:.0%}, IoU {w['iou']:.2f}, F1 {w['f1']:.2f}") c = m["car"] print(f" car {c['objects']:>5} cars | {c['on_road']:.0%} on a BGT road or parking surface") own = sweep[(sweep.site == key) & sweep.cls.isin(HEADLINE)].groupby("cls").f1.max() print(" best F1 this site could have reached with its own settings: " + ", ".join(f"{c} {own[c]:.2f}" for c in HEADLINE)) section("what SAM adds: the Grounding DINO boxes alone as building outlines") for key, s in SITES.items(): sel = final[key]["building"] sb = score("building", sel.assign(geometry=gpd.GeoSeries.from_wkt(sel["box"], crs=CRS)), s) metrics[key]["building_boxes_only"] = sb sm = metrics[key]["building"] print(f"{s['name']:<24} boxes: pixel IoU {sb['iou']:.2f}, F1 {sb['f1']:.2f}, object F1 {sb['obj_f1']:.2f} | " f"SAM 2.1 masks: pixel IoU {sm['iou']:.2f}, F1 {sm['f1']:.2f}, object F1 {sm['obj_f1']:.2f}") section("Parkwijk: share of BGT buildings found, by footprint size") s = SITES[TEST] ious, _, used_r = match_objects(clip(final[TEST]["building"], s), s["ref"]["building"]) ref_b = s["ref"]["building"].assign(found=used_r, area=s["ref"]["building"].area) ref_b["size"] = pd.cut(ref_b.area, [0, 20, 50, 150, 500, 1e9], labels=["under 20 m²", "20-50 m²", "50-150 m²", "150-500 m²", "over 500 m²"]) print(ref_b.groupby("size", observed=True).agg(buildings=("found", "size"), recall=("found", "mean")).round(2) .to_string()) print(f"median IoU of the matched buildings: {np.median(ious):.2f}") # --- Step 8: GeoPackage and metrics ------------------------------------------------------------------------ section("outputs") gpkg = OUT / "text_prompt_sam.gpkg" gpkg.unlink(missing_ok=True) for key in SITES: for cls, g in final[key].items(): if len(g): g.drop(columns="box").to_file(gpkg, layer=f"{key}_{cls}", driver="GPKG") print(f"wrote {sum(len(g) for f in final.values() for g in f.values())} polygons in " f"{len(SITES) * len(chosen)} layers to {gpkg.name} (EPSG:28992)") (OUT / "metrics.json").write_text(json.dumps(dict(chosen=chosen, metrics=metrics), indent=1, default=float)) # --- Step 9: figures ------------------------------------------------------------------------------------- COL = {"building": "#e8590c", "tree": "#2b8a3e", "water": "#1c7ed6", "car": "#f2c94c"} def show(ax, s, window=None): ax.imshow(s["rgb"], extent=(s["bounds"][0], s["bounds"][2], s["bounds"][1], s["bounds"][3])) if window: ax.set_xlim(window[0], window[2]) ax.set_ylim(window[1], window[3]) ax.set_xticks([]) ax.set_yticks([]) def save(fig, name: str) -> None: for ax in fig.axes: # GeoPandas labels map axes "Easting"/"Northing"; the scale bar says enough if not ax.get_xticks().size: ax.set_xlabel("") ax.set_ylabel("") fig.savefig(OUT / name) plt.close(fig) def scalebar(ax, metres, s, window=None): x0, y0, x1, _ = window or s["bounds"] w = x1 - x0 ax.plot([x0 + 0.04 * w, x0 + 0.04 * w + metres], [y0 + 0.04 * w] * 2, color="white", lw=3) ax.text(x0 + 0.04 * w + metres / 2, y0 + 0.06 * w, f"{metres} m", color="white", ha="center", fontsize=8, fontweight="bold") # 1. the two sites with their BGT reference fig, axes = plt.subplots(1, 2, figsize=(11, 5.8)) for ax, (key, s) in zip(axes, SITES.items()): show(ax, s) s["ref"]["building"].boundary.plot(ax=ax, color=COL["building"], lw=0.5) s["ref"]["water"].boundary.plot(ax=ax, color="#4dabf7", lw=1.2) s["ref"]["tree"].plot(ax=ax, color="#69db7c", markersize=1.5) scalebar(ax, 100, s) ax.set_title(f"{s['name']} ({s['role']}): {len(s['ref']['building'])} buildings, " f"{len(s['ref']['tree'])} registered trees", fontsize=9) fig.legend(handles=[Line2D([], [], color=COL["building"], label="BGT building (pand)"), Line2D([], [], color="#4dabf7", label="BGT water"), Line2D([], [], color="#69db7c", marker="o", ls="", ms=3, label="BGT tree")], loc="lower center", ncol=3, frameon=False) fig.suptitle("Two 512 m sites of 25 cm aerial photography (PDOK Luchtfoto 2025) and the BGT reference") save(fig, "01-sites-bgt-reference.png") # 2. the pipeline on one window: prompt -> boxes -> masks -> polygons s = SITES[CAL] cx, cy = (s["bounds"][0] + s["bounds"][2]) / 2, (s["bounds"][1] + s["bounds"][3]) / 2 win = (cx - 250, cy - 250, cx - 90, cy - 90) wbox = box(*win) v = chosen["building"] r = results[CAL] b_all = r[(r.cls == "building") & (r.prompt == v["prompt"]) & (r.det_tile == v["det_tile"])] b_sel = final[CAL]["building"] fig, axes = plt.subplots(1, 3, figsize=(12, 4.4)) show(axes[0], s, win) for _, row in b_all[b_all.intersects(wbox)].iterrows(): bx = shapely.from_wkt(row["box"]).bounds kept = row["score"] >= v["thr"] axes[0].add_patch(Rectangle(bx[:2], bx[2] - bx[0], bx[3] - bx[1], fill=False, lw=1.2 if kept else 0.7, ec=COL["building"] if kept else "white", ls="-" if kept else ":")) axes[0].set_title(f"1. Grounding DINO boxes for '{v['prompt']}'\nsolid: score >= {v['thr']}; dotted: below") show(axes[1], s, win) b_sel[b_sel.intersects(wbox)].plot(ax=axes[1], color=COL["building"], alpha=0.55, ec="white", lw=0.5) axes[1].set_title("2. SAM 2.1 traces the object\nin each kept box") show(axes[2], s, win) s["ref"]["building"].boundary.plot(ax=axes[2], color="cyan", lw=0.8) b_sel[b_sel.intersects(wbox)].boundary.plot(ax=axes[2], color=COL["building"], lw=1.2) axes[2].set_title("3. Polygons (orange) against\nBGT pand (cyan)") for ax in axes: scalebar(ax, 25, s, win) save(fig, "02-pipeline-boxes-masks-polygons.png") # 3. all four prompts on each site fig, axes = plt.subplots(1, 2, figsize=(11, 5.8)) for ax, (key, s) in zip(axes, SITES.items()): show(ax, s) for cls in ["water", "tree", "building", "car"]: if len(final[key][cls]): final[key][cls].plot(ax=ax, color=COL[cls], alpha=0.6, lw=0) scalebar(ax, 100, s) ax.set_title(s["name"], fontsize=9) fig.legend(handles=[Patch(color=COL[c], alpha=0.7, label=f"'{v['prompt']}' ({v['det_tile']} px, >= {v['thr']})") for c, v in chosen.items()], loc="lower center", ncol=4, frameon=False) fig.suptitle("Four text prompts with the settings chosen on Kanaleneiland") save(fig, "03-four-prompts-both-sites.png") # 4. wording and tile size fig, axes = plt.subplots(1, 2, figsize=(11, 3.4), gridspec_kw={"width_ratios": [3, 3]}) for ax, cls in zip(axes, ["building", "tree"]): d = best_each[best_each.cls == cls].pivot(index="prompt", columns="det_tile", values="f1") d = d.loc[CANDIDATES[cls]] d.plot.barh(ax=ax, color={1024: "#adb5bd", 512: "#495057"}, width=0.7) ax.set_xlabel("best F1 on Kanaleneiland") ax.set_ylabel("") ax.set_title(f"{cls}: {'pixel F1' if cls == 'building' else 'crown F1'}") ax.legend(title="tile (px)", fontsize=8, loc="lower right") ax.invert_yaxis() fig.suptitle("The same objects, worded differently and cut into different tile sizes") save(fig, "04-prompt-wording-tile-size.png") # 5. threshold sweep for the chosen settings TITLES = {"building": "building: pixel F1", "tree": "tree: crown F1", "water": "water: pixel F1"} fig, axes = plt.subplots(1, 3, figsize=(12, 3.6), sharey=True) for ax, cls in zip(axes, HEADLINE): v = chosen[cls] for key, s in SITES.items(): d = sweep[(sweep.site == key) & (sweep.cls == cls) & (sweep.prompt == v["prompt"]) & (sweep.det_tile == v["det_tile"])] ax.plot(d.thr, d.f1, marker="o", ms=3, label=s["name"], ls="-" if key == CAL else "--") ax.axvline(v["thr"], color="grey", lw=0.8) ax.set_title(f"{TITLES[cls]}\n'{v['prompt']}', {v['det_tile']}-px tiles") ax.set_xlabel("Grounding DINO box score threshold") ax.grid(alpha=0.3) axes[0].set_ylabel("F1") axes[0].legend(fontsize=8) fig.suptitle("Choose the threshold on one site (grey line), then check it on another") save(fig, "05-threshold-sweep.png") # 6. building errors on the test site s = SITES[TEST] pred_b = clip(final[TEST]["building"], s) _, used_p, used_r = match_objects(pred_b, s["ref"]["building"]) fig, ax = plt.subplots(figsize=(7.5, 7.5)) show(ax, s) s["ref"]["building"][~used_r].plot(ax=ax, color="none", ec="#e03131", lw=0.9) if used_p.any(): pred_b[used_p].plot(ax=ax, color="#40c057", alpha=0.5, lw=0) if (~used_p).any(): pred_b[~used_p].plot(ax=ax, color="#fab005", alpha=0.6, lw=0) scalebar(ax, 100, s) ax.legend(handles=[Patch(color="#40c057", alpha=0.6, label=f"matches one BGT building (IoU >= 0.5): {used_p.sum()}"), Patch(color="#fab005", alpha=0.7, label=f"polygon with no match: {(~used_p).sum()}"), Patch(fc="none", ec="#e03131", label=f"BGT building not matched: {(~used_r).sum()}")], loc="upper left", fontsize=8, framealpha=0.85) mb = metrics[TEST]["building"] ax.set_title(f"{s['name']}, '{chosen['building']['prompt']}': pixel F1 {mb['f1']:.2f}, " f"object F1 {mb['obj_f1']:.2f}") save(fig, "06-parkwijk-building-errors.png") print("figures written to output/")