From 5ee3bb998ca015119a6cd5e3d1e8ffbfebd2c5a3 Mon Sep 17 00:00:00 2001 From: Jean-Pierre Martineau Date: Mon, 13 Jul 2026 21:24:29 -0400 Subject: [PATCH] Activity charts: bucket by local time in SQL, storing UTC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Time Machine's day / hour / quarter-hour activity feeds now group on datetime(observed_at, 'localtime') in SQLite — rotating each stored-UTC timestamp into the machine's zone per-instant (DST-correct) before the key is sliced — so bars break on local midnight and local hour boundaries in any zone, half-hour offsets included, not on UTC edges. The window 'since' filter stays on the raw UTC column: the window is an absolute span, only the labelling is local. The old caller-side current-offset histogram rotation is gone, and _fill_days / _fill_hours and the axis tick labels now walk local wall-clock keys to match. Tests derive their expected local buckets through the same rotation so they hold in any runner zone. Co-Authored-By: Claude Opus 4.8 --- meshterm/persistence/repository.py | 67 ++++++++++++++++++------------ meshterm/ui/timemachine_screen.py | 62 ++++++++++++++------------- tests/test_dashboard.py | 11 +++-- tests/test_timemachine.py | 41 +++++++++++------- 4 files changed, 108 insertions(+), 73 deletions(-) diff --git a/meshterm/persistence/repository.py b/meshterm/persistence/repository.py index 432b22c..b875487 100644 --- a/meshterm/persistence/repository.py +++ b/meshterm/persistence/repository.py @@ -833,16 +833,21 @@ class Repository: def daily_activity(self) -> list[tuple[str, int, int]]: """Per-day activity totals across the whole stored history, oldest first. - Days are UTC calendar days (``observed_at`` is stored as UTC ISO-8601, so the - grouping is a cheap string prefix); the Time Machine labels them as such. + Days are **local** calendar days: ``observed_at`` is stored as UTC ISO-8601, + but SQLite's ``datetime(…, 'localtime')`` rotates each timestamp into the + machine's zone (per-instant, so DST-correct) before the day prefix is sliced, + so a bar breaks at local midnight — not at the UTC-offset hour. Read the same + database in another zone and the days re-bucket to wherever you are, which is + the point: history is shown in the viewer's local time. Returns: - ``(day, packets, nodes)`` per day with any activity: the day as + ``(day, packets, nodes)`` per day with any activity: the day as a local ``YYYY-MM-DD``, every stored observation counted, and the distinct identified nodes heard (``packet`` rows excluded — no reliable identity). """ rows = self._conn.execute( - "SELECT substr(observed_at, 1, 10) AS day, COUNT(*) AS pkts, " + "SELECT substr(datetime(observed_at, 'localtime'), 1, 10) AS day, " + "COUNT(*) AS pkts, " "COUNT(DISTINCT CASE WHEN kind != 'packet' THEN node END) AS nodes " "FROM observations GROUP BY day ORDER BY day" ).fetchall() @@ -852,24 +857,28 @@ class Repository: """Per-clock-hour activity totals since a time, oldest first (the 24 h window feed). The hour-resolution sibling of :meth:`daily_activity`: the same packet and - distinct-node counts, but grouped on the ISO-8601 ``observed_at`` prefix down - to the hour (``YYYY-MM-DDTHH``, a cheap string slice — position 11 is the - ``T``) so a day of history folds into ~24 rows however dense it is. Only hours + distinct-node counts, grouped down to the hour. ``observed_at`` is stored as + UTC, so SQLite's ``datetime(…, 'localtime')`` rotates each timestamp into the + machine's zone before the ``YYYY-MM-DDTHH`` key is sliced — the buckets break + on local hour boundaries in every zone, half-hour offsets included, not on UTC + edges. A day of history folds into ~24 rows however dense it is. Only hours with traffic come back; the caller fills the quiet ones (see :func:`~meshterm.ui.timemachine_screen._fill_hours`) so the axis is real - clock time. + clock time. The ``since`` filter stays on the raw UTC column — the window is + an absolute span; only the labelling is local. Args: since: Only observations at or after this time. Returns: - ``(hour, packets, nodes)`` per active hour: ``hour`` as a UTC + ``(hour, packets, nodes)`` per active hour: ``hour`` as a local ``YYYY-MM-DDTHH``, every observation counted, and the distinct identified nodes heard (``packet`` rows excluded from the node count — no reliable identity), oldest first. """ rows = self._conn.execute( - "SELECT substr(observed_at, 1, 13) AS hour, COUNT(*) AS pkts, " + "SELECT replace(substr(datetime(observed_at, 'localtime'), 1, 13), ' ', 'T') " + "AS hour, COUNT(*) AS pkts, " "COUNT(DISTINCT CASE WHEN kind != 'packet' THEN node END) AS nodes " "FROM observations WHERE observed_at >= ? GROUP BY hour ORDER BY hour", (since.isoformat(),), @@ -877,22 +886,25 @@ class Repository: return [(row["hour"], int(row["pkts"]), int(row["nodes"])) for row in rows] def hourly_activity(self, *, since: Optional[datetime] = None) -> list[int]: - """Observation counts by UTC hour of day (0–23) across the stored history. + """Observation counts by local hour of day (0–23) across the stored history. The whole-mesh Rhythm chart's feed: every stored observation counted into - the hour-of-day it arrived, grouped in SQL off a cheap string slice of the - ISO-8601 ``observed_at`` (position 12 is where ``HH`` starts), so the cost - stays 24 rows however deep the history grows. The caller rotates the - histogram into local hours — one current-offset rotation, which is honest - enough for a rhythm chart even across a DST boundary. + the hour-of-day it arrived. ``observed_at`` is stored as UTC, so SQLite's + ``datetime(…, 'localtime')`` rotates each timestamp into the machine's zone + (per-instant, so DST-correct) before the ``HH`` slice, giving a histogram + already in local hours — no caller rotation needed. The cost stays 24 rows + however deep the history grows. Args: since: Only observations at or after this time, if given. Returns: - 24 counts, index = UTC hour. + 24 counts, index = local hour. """ - sql = "SELECT substr(observed_at, 12, 2) AS hh, COUNT(*) AS n FROM observations" + sql = ( + "SELECT substr(datetime(observed_at, 'localtime'), 12, 2) AS hh, " + "COUNT(*) AS n FROM observations" + ) params: list[Any] = [] if since is not None: sql += " WHERE observed_at >= ?" @@ -907,25 +919,26 @@ class Repository: return counts def quarter_hour_activity(self, *, since: Optional[datetime] = None) -> list[int]: - """Observation counts by UTC quarter-hour of day (0–95) across the history. + """Observation counts by local quarter-hour of day (0–95) across the history. The finer sibling of :meth:`hourly_activity` — four slices an hour instead of one — feeding the whole-mesh Rhythm chart at its full 15-minute resolution. The - slice index is ``HH * 4 + MM // 15``, computed in SQL off the same cheap ISO-8601 - substrings (``HH`` at position 12, ``MM`` at position 15), so the cost stays 96 - rows however deep the history grows. The caller rotates the histogram into local - time (one current-offset rotation, in 15-minute units). + slice index is ``HH * 4 + MM // 15``, computed in SQL off the ``HH``/``MM`` + substrings of ``datetime(…, 'localtime')`` (``observed_at`` is stored as UTC, + rotated per-instant into the machine's zone before slicing, so DST-correct), so + the histogram lands in local time with no caller rotation. The cost stays 96 + rows however deep the history grows. Args: since: Only observations at or after this time, if given. Returns: - 96 counts, index = UTC quarter-hour slice of the day. + 96 counts, index = local quarter-hour slice of the day. """ sql = ( - "SELECT CAST(substr(observed_at, 12, 2) AS INTEGER) * 4 " - "+ CAST(substr(observed_at, 15, 2) AS INTEGER) / 15 AS slot, COUNT(*) AS n " - "FROM observations" + "SELECT CAST(substr(datetime(observed_at, 'localtime'), 12, 2) AS INTEGER) " + "* 4 + CAST(substr(datetime(observed_at, 'localtime'), 15, 2) AS INTEGER) " + "/ 15 AS slot, COUNT(*) AS n FROM observations" ) params: list[Any] = [] if since is not None: diff --git a/meshterm/ui/timemachine_screen.py b/meshterm/ui/timemachine_screen.py index b1bb9a1..4d49e23 100644 --- a/meshterm/ui/timemachine_screen.py +++ b/meshterm/ui/timemachine_screen.py @@ -22,7 +22,7 @@ transmits. from __future__ import annotations -from datetime import date, datetime, timedelta, timezone +from datetime import date, datetime, timedelta from statistics import median from typing import TYPE_CHECKING, Callable, Optional @@ -464,7 +464,9 @@ def _day_ticks(shown: list, chars: int) -> list[tuple[int, str]]: """ n = len(shown) centers = _day_centers(n, chars) - today = utcnow().strftime("%Y-%m-%d") + # The day keys are local calendar days (see Repository.daily_activity), so "today" + # is the local date too — datetime.now() is naive local, exactly what we compare. + today = datetime.now().strftime("%Y-%m-%d") def label_of(picks: list[int]) -> list[str]: labels: list[str] = [] @@ -496,13 +498,14 @@ def _hour_ticks(shown: list, chars: int) -> list[tuple[int, str]]: The hour-resolution sibling of :func:`_day_ticks` for the 24 h window: one tick per hour where the labels fit, else an evenly spaced subset keeping both ends (see :func:`_fit_ticks`), each label centred under its own bar (see - :func:`_day_centers`). Labels read as local ``HH:00`` clock times — the UTC hour - buckets rotated into the viewer's zone — and the newest bar reads ``now``, - mirroring the node page's closing ``now``. + :func:`_day_centers`). The hour keys are already local (see + :meth:`~meshterm.persistence.repository.Repository.hourly_series`), so a label is + just its key's ``HH:00``, no conversion; the newest bar reads ``now``, mirroring + the node page's closing ``now``. Args: - shown: The charted hours, oldest first, each ``(hour_iso, ...)`` in UTC - (``YYYY-MM-DDTHH``). + shown: The charted hours, oldest first, each ``(hour_iso, ...)`` a local + ``YYYY-MM-DDTHH``. chars: The chart's width in character cells. Returns: @@ -518,13 +521,11 @@ def _hour_ticks(shown: list, chars: int) -> list[tuple[int, str]]: labels.append("now") continue try: - hour = datetime.strptime(shown[i][0], "%Y-%m-%dT%H").replace( - tzinfo=timezone.utc - ) + hour = datetime.strptime(shown[i][0], "%Y-%m-%dT%H") except ValueError: labels.append(shown[i][0]) continue - labels.append(f"{hour.astimezone():%H:00}") + labels.append(f"{hour:%H:00}") return labels return _fit_ticks(n, chars, centers, label_of) @@ -553,10 +554,12 @@ def _fill_days( if not active: return [] by_iso = {iso: (packets, nodes) for iso, packets, nodes in active} + # The keys are local calendar days, so bound the fill by local dates too — .date() + # on the raw UTC-aware since/now would floor a day early in western zones. first = date.fromisoformat(active[0][0]) - floor = since.date() if since is not None else first + floor = since.astimezone().date() if since is not None else first start = max(first, floor) - end = max(start, now.date()) + end = max(start, now.astimezone().date()) out: list[tuple[str, int, int]] = [] day = start while day <= end: @@ -575,28 +578,34 @@ def _fill_hours( The hour-resolution sibling of :func:`_fill_days`, feeding the 24 h window's charts: :meth:`~meshterm.persistence.repository.Repository.hourly_series` returns only hours with traffic, so a silent hour would fuse its busy neighbours. This - walks every UTC clock hour of the window and emits ``(iso, 0, 0)`` for the quiet + walks every local clock hour of the window and emits ``(iso, 0, 0)`` for the quiet ones, so the x-axis is real clock time. The range runs from the window's floor (but never earlier than the first hour recorded — we don't invent emptiness from before monitoring began) through the current hour. + The keys are local wall-clock ``YYYY-MM-DDTHH`` (matching the SQL grouping), so the + walk steps a *naive* local clock: adding an hour advances the wall clock, which is + DST-robust — a spring-forward gap fills as an empty bar and a fall-back repeat sums + into one key, exactly as the grouping already did. + Args: active: ``(hour_iso, packets, nodes)`` for hours with activity, oldest first, - each ``hour_iso`` a UTC ``YYYY-MM-DDTHH``. + each ``hour_iso`` a local ``YYYY-MM-DDTHH``. since: The window's start. now: The current time (the series ends on its clock hour). Returns: - ``(hour_iso, packets, nodes)`` for every UTC clock hour in range, oldest first. + ``(hour_iso, packets, nodes)`` for every local clock hour in range, oldest + first. """ if not active: return [] def floor_hour(when: datetime) -> datetime: - return when.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0) + return when.astimezone().replace(tzinfo=None, minute=0, second=0, microsecond=0) by_iso = {iso: (packets, nodes) for iso, packets, nodes in active} - first = datetime.strptime(active[0][0], "%Y-%m-%dT%H").replace(tzinfo=timezone.utc) + first = datetime.strptime(active[0][0], "%Y-%m-%dT%H") start = max(first, floor_hour(since)) end = max(start, floor_hour(now)) out: list[tuple[str, int, int]] = [] @@ -677,12 +686,9 @@ def _mesh_sections( # The mesh-wide rhythm (charted below) folds the whole window into 96 fifteen-minute # slices, so a busy slice's tally can top any single day's — compute it up front so its - # peak joins the day peaks in sizing one shared y-axis gutter. - offset_slots = round( - (datetime.now().astimezone().utcoffset() or timedelta()).total_seconds() / 900 - ) - utc_slots = ctx.repo.quarter_hour_activity(since=since) - slots = [utc_slots[(s - offset_slots) % 96] for s in range(96)] + # peak joins the day peaks in sizing one shared y-axis gutter. The slices come back + # already in local time (rotated per-instant in SQL), so no offset shuffle here. + slots = ctx.repo.quarter_hour_activity(since=since) # The y-axis gutter is sized from the whole window's peaks (not just the visible # slice) and shared by every chart, so all their gutters — and thus their left edges — @@ -701,7 +707,7 @@ def _mesh_sections( ("Packets per hour", "Nodes per hour") if hourly else ("Packets per day", "Nodes per day") ) - out.append(_heading(pkt_title, "local hours" if hourly else "UTC days")) + out.append(_heading(pkt_title, "local hours" if hourly else "local days")) out.extend( axis_chart( timeline_rows(_day_columns(packets, chars), rows=_CHART_ROWS), @@ -720,9 +726,9 @@ def _mesh_sections( ) # The node page's rhythm chart, mesh-wide and four times finer: when does this *mesh* - # talk? The 96 fifteen-minute slices (grouped by UTC in SQL, rotated into local time - # above) keep their own 48-cell width but share the day charts' gutter, so this chart's - # left edge lines up with the two above it. + # talk? The 96 fifteen-minute slices (grouped in local time by SQL) keep their own + # 48-cell width but share the day charts' gutter, so this chart's left edge lines up + # with the two above it. out.append(Text()) out.append(_heading("Rhythm", "packets by local time of day · 15-min slices")) out.extend( diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 2dcd040..d66d4ce 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -8,7 +8,7 @@ covered in ``test_braillechart``.) from __future__ import annotations import re -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from pathlib import Path from meshterm.core.events import MeshEvent @@ -309,14 +309,19 @@ def test_recent_observations_rehydrate_the_packet_payload_class(tmp_path: Path) def test_quarter_hour_activity_buckets_by_15_minute_slice(tmp_path: Path) -> None: - """Observations fall into 96 quarter-hour-of-day slots (``HH * 4 + MM // 15``).""" + """Observations fall into 96 quarter-hour-of-day slots (``HH * 4 + MM // 15``). + + The slots are local time-of-day, so the observations are stamped at local + wall-clock instants (naive → ``.astimezone()``) — the slice index then matches + each ``HH:MM`` directly, in any runner zone. + """ repo = Repository(tmp_path / "q.db") run = repo.start_run("monitor", {}, None) def at(hh: int, mm: int) -> Observation: return Observation( node="n", name="n", kind="advert", - observed_at=datetime(2026, 7, 8, hh, mm, tzinfo=timezone.utc), + observed_at=datetime(2026, 7, 8, hh, mm).astimezone(), ) for hh, mm in [(0, 0), (0, 44), (17, 55), (17, 59)]: diff --git a/tests/test_timemachine.py b/tests/test_timemachine.py index 5aef0e4..4a7716a 100644 --- a/tests/test_timemachine.py +++ b/tests/test_timemachine.py @@ -110,8 +110,8 @@ def test_daily_activity_counts_packets_and_nodes(tmp_path: Path) -> None: repo.close() -def test_hourly_activity_groups_by_utc_hour(tmp_path: Path) -> None: - """The rhythm feed lands each observation in its UTC hour, windowed on demand.""" +def test_hourly_activity_groups_by_local_hour(tmp_path: Path) -> None: + """The rhythm feed lands each observation in its local hour, windowed on demand.""" repo = Repository(tmp_path / "hh.db") run = repo.start_run("monitor", {}, None) base = utcnow().replace(hour=5, minute=0, second=0, microsecond=0) @@ -122,14 +122,18 @@ def test_hourly_activity_groups_by_utc_hour(tmp_path: Path) -> None: repo.record_observation( run, Observation(node="aa" * 6, observed_at=base.replace(hour=17)) ) + # Each UTC instant is rotated into the machine's zone before bucketing; derive the + # expected local hours the same way so the test holds in any zone. + early, late = base.astimezone().hour, base.replace(hour=17).astimezone().hour counts = repo.hourly_activity() - assert counts[5] == 3 and counts[17] == 1 and sum(counts) == 4 + assert counts[early] == 3 and counts[late] == 1 and sum(counts) == 4 + # The window filter is on the raw UTC column, so only the 17:00 UTC obs survives. assert sum(repo.hourly_activity(since=base.replace(hour=6))) == 1 repo.close() def test_hourly_series_groups_by_clock_hour(tmp_path: Path) -> None: - """The 24 h feed folds observations into UTC clock-hour buckets: packets and nodes.""" + """The 24 h feed folds observations into local clock-hour buckets: packets and nodes.""" repo = Repository(tmp_path / "hs.db") run = repo.start_run("monitor", {}, None) now = utcnow() @@ -144,8 +148,11 @@ def test_hourly_series_groups_by_clock_hour(tmp_path: Path) -> None: repo.record_observation(run, Observation(node="aa" * 6, observed_at=base + timedelta(minutes=1))) series = repo.hourly_series(now - timedelta(days=1)) by_hour = {iso: (pkts, nodes) for iso, pkts, nodes in series} - assert by_hour[three_h.strftime("%Y-%m-%dT%H")] == (4, 2) # 3 adverts + a packet row; 2 nodes - assert by_hour[base.strftime("%Y-%m-%dT%H")] == (1, 1) # just this hour's lone advert + # Keys are local wall-clock hours, so form the expected keys via the same rotation. + three_h_key = three_h.astimezone().strftime("%Y-%m-%dT%H") + base_key = base.astimezone().strftime("%Y-%m-%dT%H") + assert by_hour[three_h_key] == (4, 2) # 3 adverts + a packet row; 2 nodes + assert by_hour[base_key] == (1, 1) # just this hour's lone advert assert series == sorted(series) # oldest first repo.close() @@ -195,13 +202,15 @@ def test_day_columns_always_hits_the_exact_width() -> None: def test_fill_days_shows_gap_days_as_zero_bars() -> None: """A quiet day between busy ones is emitted with zero counts, not skipped.""" - from datetime import datetime, timezone + from datetime import datetime from meshterm.ui.timemachine_screen import _fill_days + # The day keys are local calendar days, so bound the fill by local wall-clock + # instants (naive → .astimezone() reads them as local) — zone-independent. active = [("2026-07-02", 50, 5), ("2026-07-05", 80, 8)] - now = datetime(2026, 7, 6, 12, tzinfo=timezone.utc) - filled = _fill_days(active, datetime(2026, 7, 1, tzinfo=timezone.utc), now) + now = datetime(2026, 7, 6, 12).astimezone() + filled = _fill_days(active, datetime(2026, 7, 1).astimezone(), now) assert [iso for iso, _p, _n in filled] == [ "2026-07-02", "2026-07-03", "2026-07-04", "2026-07-05", "2026-07-06", ] @@ -212,26 +221,28 @@ def test_fill_days_shows_gap_days_as_zero_bars() -> None: def test_fill_days_never_invents_days_before_recording_began() -> None: """A window floor earlier than the first recorded day clamps to that day, not the floor.""" - from datetime import datetime, timezone + from datetime import datetime from meshterm.ui.timemachine_screen import _fill_days active = [("2026-07-10", 10, 1)] - now = datetime(2026, 7, 12, tzinfo=timezone.utc) - filled = _fill_days(active, datetime(2026, 6, 12, tzinfo=timezone.utc), now) # a 30 d floor + now = datetime(2026, 7, 12, 12).astimezone() + filled = _fill_days(active, datetime(2026, 6, 12).astimezone(), now) # a 30 d floor assert filled[0][0] == "2026-07-10" # not the 2026-06-12 floor assert filled[-1][0] == "2026-07-12" # …but still runs through today def test_fill_hours_shows_gap_hours_as_zero_bars() -> None: """A quiet hour between busy ones is emitted with zero counts, not skipped.""" - from datetime import datetime, timezone + from datetime import datetime from meshterm.ui.timemachine_screen import _fill_hours + # Hour keys are local wall-clock, so bound the fill by local instants (naive → + # .astimezone() reads them as local), keeping the test zone-independent. active = [("2026-07-12T08", 30, 3), ("2026-07-12T11", 20, 2)] - since = datetime(2026, 7, 12, 6, 30, tzinfo=timezone.utc) # floors to 06:00… - now = datetime(2026, 7, 12, 12, 15, tzinfo=timezone.utc) # …but the first hour recorded wins + since = datetime(2026, 7, 12, 6, 30).astimezone() # floors to 06:00… + now = datetime(2026, 7, 12, 12, 15).astimezone() # …but the first hour recorded wins filled = _fill_hours(active, since, now) assert [iso for iso, _p, _n in filled] == [f"2026-07-12T{h:02d}" for h in range(8, 13)] counts = {iso: pkts for iso, pkts, _n in filled}