Tile decode: decide the layer once, on the way past
The narrowing walked each layer twice -- once for _layer_head to read its name, once to decode it -- which is free when there is a lot to skip and a straight loss when there isn't. Measured against a cache spanning zooms 2-14: low-zoom tiles carry no buildings, housenumbers or POIs at all (a z10 tile is 98% drawn layers, a z2 tile 100%), so the whitelist saved them nothing while the extra walk cost 16-18%. The name is field 1 and in practice the first thing written, so the walk already knows by then whether the rest is worth keeping; from there an unwanted layer is stepped over rather than copied out. Field order isn't guaranteed though, so the flag is only an optimisation -- the decision is the check after the loop, which is right however the fields arrived and also covers a layer declaring no name at all. _layer_head goes away. On the regular platform this turns a 0.97x into a 1.03x on low-zoom tiles and 1.00x on the largest, while urban z14 tiles keep their 3.19x; the whole 293-tile cache goes 1.4x -> 1.6x. On the Lyra the median tile is 3.24x and the largest 4.85x, both up. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
eed7f6b38e
commit
71e979eadb
1 file changed
+26
-38
+26
-38
@@ -239,29 +239,50 @@ def _decode_feature(buf: bytes, keys: list[str], values: list[Any]) -> Optional[
|
||||
return Feature(geom_type=geom_type, rings=_decode_geometry(geom_ints), tags=tags)
|
||||
|
||||
|
||||
def _decode_layer(buf: bytes) -> Layer:
|
||||
"""Decode a single Layer message and all of its features."""
|
||||
def _decode_layer(buf: bytes, wanted: Optional[Container[str]] = None) -> Layer:
|
||||
"""Decode a single Layer message, and its features unless the caller doesn't want them.
|
||||
|
||||
The name arrives in field 1, which in practice is the first thing written, so once it
|
||||
has been read the walk already knows whether any of the rest is worth keeping — from
|
||||
there an unwanted layer is stepped over rather than copied out. Protobuf permits any
|
||||
field order though, so ``skipping`` is only ever an optimisation: the decision is the
|
||||
check after the loop, which is correct however the fields arrived (and covers a layer
|
||||
that declares no name at all).
|
||||
|
||||
Args:
|
||||
buf: The Layer message's bytes.
|
||||
wanted: Layer names worth decoding features for; ``None`` decodes every layer.
|
||||
|
||||
Returns:
|
||||
The layer. One the caller didn't ask for comes back named and empty.
|
||||
"""
|
||||
r = _Reader(buf)
|
||||
name = ""
|
||||
extent = 4096
|
||||
keys: list[str] = []
|
||||
values: list[Any] = []
|
||||
feature_blobs: list[bytes] = []
|
||||
skipping = False
|
||||
while not r.eof():
|
||||
field_no, wire = r.tag()
|
||||
if field_no == 1 and wire == 2:
|
||||
name = r.blob().decode("utf-8", "ignore")
|
||||
skipping = wanted is not None and name not in wanted
|
||||
elif field_no == 5 and wire == 0:
|
||||
extent = r.varint()
|
||||
elif skipping:
|
||||
r.skip(wire)
|
||||
elif field_no == 2 and wire == 2:
|
||||
feature_blobs.append(r.blob())
|
||||
elif field_no == 3 and wire == 2:
|
||||
keys.append(r.blob().decode("utf-8", "ignore"))
|
||||
elif field_no == 4 and wire == 2:
|
||||
values.append(_decode_value(r.blob()))
|
||||
elif field_no == 5 and wire == 0:
|
||||
extent = r.varint()
|
||||
else:
|
||||
r.skip(wire)
|
||||
layer = Layer(name=name, extent=extent)
|
||||
if wanted is not None and name not in wanted:
|
||||
return layer
|
||||
for blob in feature_blobs:
|
||||
feat = _decode_feature(blob, keys, values)
|
||||
if feat is not None:
|
||||
@@ -269,34 +290,6 @@ def _decode_layer(buf: bytes) -> Layer:
|
||||
return layer
|
||||
|
||||
|
||||
def _layer_head(buf: bytes) -> tuple[str, int]:
|
||||
"""Read a Layer message's identity — ``(name, extent)`` — without decoding features.
|
||||
|
||||
Walks the layer's top-level fields, stepping over the features and the string pools
|
||||
instead of building them. Protobuf fields may appear in any order, so this reads to
|
||||
the end rather than stopping at the first name; every field it passes is a varint or
|
||||
a length-skip, so the walk stays cheap even on a layer holding thousands of features.
|
||||
|
||||
Args:
|
||||
buf: The Layer message's bytes.
|
||||
|
||||
Returns:
|
||||
The layer's name (empty if it declares none) and its coordinate extent.
|
||||
"""
|
||||
r = _Reader(buf)
|
||||
name = ""
|
||||
extent = 4096
|
||||
while not r.eof():
|
||||
field_no, wire = r.tag()
|
||||
if field_no == 1 and wire == 2:
|
||||
name = r.blob().decode("utf-8", "ignore")
|
||||
elif field_no == 5 and wire == 0:
|
||||
extent = r.varint()
|
||||
else:
|
||||
r.skip(wire)
|
||||
return name, extent
|
||||
|
||||
|
||||
def decode_tile(data: bytes, *, layers: Optional[Container[str]] = None) -> list[Layer]:
|
||||
"""Decode a vector tile (optionally gzip-compressed) into its layers.
|
||||
|
||||
@@ -320,12 +313,7 @@ def decode_tile(data: bytes, *, layers: Optional[Container[str]] = None) -> list
|
||||
while not r.eof():
|
||||
field_no, wire = r.tag()
|
||||
if field_no == 3 and wire == 2: # Tile.layers
|
||||
blob = r.blob()
|
||||
if layers is None:
|
||||
out.append(_decode_layer(blob))
|
||||
continue
|
||||
name, extent = _layer_head(blob)
|
||||
out.append(_decode_layer(blob) if name in layers else Layer(name=name, extent=extent))
|
||||
out.append(_decode_layer(r.blob(), layers))
|
||||
else:
|
||||
r.skip(wire)
|
||||
return out
|
||||
|
||||
Reference in new issue
Block a user