From 039e91b4e44af94ff020a3b349c63b117ba027e3 Mon Sep 17 00:00:00 2001 From: Chris Amow Date: Mon, 10 Aug 2026 17:10:24 -0500 Subject: [PATCH] Stop a zero-price tick and a late exchange bar corrupting the store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two faults, both introduced by the tick feed, both visible as a huge bar that flattened the price scale. A LEVEL_ONE_FUTURES update arrived carrying LAST_PRICE: 0. The parser rejected None, but 0 is not None, so a minute opened at zero — o=0.0 h=7777.25 l=0.0 — and provisional_higher carried that low into 5m, 15m, 30m, 1h and the daily bar. Non-positive prices are treated as absent now, so the last real price carries forward and the update still counts as the trade it is. Separately, store.put replaced a bar only when it matched the tail. That was sufficient while one closed bar arrived per minute, but ticks open the next minute before CHART_FUTURES delivers the previous one, so the exchange's own bar stopped matching the tail and was silently dropped — leaving the tick-built approximation, with its partial volume, in place permanently. put now searches back a bounded number of buckets for the one it belongs to, and refuses to let a provisional bar overwrite a settled one. Tests cover all three invariants: a zero price parses as a trade with no price, a late closed bar replaces its bucket and keeps the exchange's volume, and a tick cannot overwrite a settled bar. Co-Authored-By: Claude Opus 5 --- app/bars/store.py | 28 +++++++++++++++++++++++++--- app/market/schwab.py | 6 ++++++ docs/IMPLEMENTATION_PLAN.md | 19 +++++++++++++++++++ tests/test_schwab_source.py | 14 ++++++++++++++ tests/test_store.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 3 deletions(-) diff --git a/app/bars/store.py b/app/bars/store.py index 45de48c..97005cd 100644 --- a/app/bars/store.py +++ b/app/bars/store.py @@ -16,12 +16,34 @@ class InMemoryBarStore: lambda: deque(maxlen=max_bars_per_tf) ) + # How far back from the tail a late bar may still land. A closed minute bar + # arrives a beat after the ticks that opened the next minute, so it is + # rarely more than a bucket or two behind. + LATE_BAR_LOOKBACK = 8 + def put(self, bar: Bar) -> None: + """Store a bar, replacing the bucket it belongs to. + + Matching only the tail was enough while one closed bar arrived per + minute. With tick-built bars a minute's authoritative bar shows up + *after* ticks have already opened the next one, so the exchange's own + figures were being dropped and the approximation left in place forever. + """ bars = self._bars[bar.tf] - if bars and bars[-1].t == bar.t: - bars[-1] = bar - elif not bars or bar.t > bars[-1].t: + if not bars or bar.t > bars[-1].t: bars.append(bar) + return + for index in range(len(bars) - 1, max(-1, len(bars) - self.LATE_BAR_LOOKBACK - 1), -1): + if bars[index].t == bar.t: + # A provisional bar must never overwrite a settled one: ticks + # keep arriving for a minute the exchange has already closed. + if bars[index].closed and not bar.closed: + return + bars[index] = bar + return + if bars[index].t < bar.t: + # Buckets are ordered, so nothing further back can match. + return def get(self, tf: Timeframe, limit: int | None = None) -> list[Bar]: bars = list(self._bars[tf]) diff --git a/app/market/schwab.py b/app/market/schwab.py index c7e7c19..76db41d 100644 --- a/app/market/schwab.py +++ b/app/market/schwab.py @@ -56,6 +56,12 @@ def parse_level_one(message: dict) -> list[tuple[int, float | None, int]]: ticks: list[tuple[int, float | None, int]] = [] for content in message.get("content") or []: price = content.get(FIELD_LAST_PRICE) + # Zero is not a price. The field arrives as 0 on some updates, and + # because 0 is not None it opened a bar at zero, which dragged the low + # of that minute — and every timeframe aggregating it — to the bottom of + # the chart. Treated as absent, so the last real price carries forward. + if price is not None and float(price) <= 0: + price = None size = content.get(FIELD_LAST_SIZE) traded_at = content.get(FIELD_TRADE_TIME) # A trade stamp alongside a moved cumulative volume is a trade even when diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 63305dc..8ed09cd 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -1526,3 +1526,22 @@ turned out to render exactly on its bar — zero pixels off at 1h, 30m and 15m because the anchor had snapped to the *drawn* timeframe extreme (the 09:00 1h low, 7744.25) while being checked against 1m bars, where it matches neither extreme. Always compare an anchor against the timeframe it was drawn on. + +**A zero price wrecked every timeframe's scale.** A LEVEL_ONE_FUTURES update +arrived with `LAST_PRICE: 0`. The parser rejected `None` but `0` is not `None`, +so a minute opened at zero — `o=0.0 h=7777.25 l=0.0` — and `provisional_higher` +carried that low into 5m, 15m, 30m, 1h and the daily bar, flattening the price +scale everywhere. Non-positive prices are now treated as absent, so the last +real price carries forward, and the tick still counts as a trade. + +**The exchange's own bars were being dropped.** `store.put` replaced a bar only +when it matched the *tail*. That held while one closed bar arrived per minute, +but ticks open the next minute before CHART_FUTURES delivers the previous one — +so the authoritative bar no longer matched the tail and was discarded, leaving +the tick approximation and its partial volume in place permanently. `put` now +searches back a bounded number of buckets, and refuses to let a provisional bar +overwrite a settled one. + +Both were introduced by the tick feature and both are covered by tests: a zero +price parses as a trade with no price, a late closed bar replaces its bucket and +keeps the exchange's volume, and a tick cannot overwrite a settled bar. diff --git a/tests/test_schwab_source.py b/tests/test_schwab_source.py index 06cabd6..438be65 100644 --- a/tests/test_schwab_source.py +++ b/tests/test_schwab_source.py @@ -279,3 +279,17 @@ def test_a_trade_known_only_by_its_volume_still_counts(): ] } assert parse_level_one(volume_only) == [(1786356932000, None, 0)] + + +def test_a_zero_last_price_is_not_a_price(): + # Seen live: LAST_PRICE arrived as 0, and because 0 is not None it opened a + # bar at zero whose low dragged every aggregating timeframe to the floor. + zero = {"content": [{"key": "/ES", "LAST_PRICE": 0, "LAST_SIZE": 2, + "TRADE_TIME_MILLIS": 1786356933000}]} + # Still a trade — size and stamp are there — but with no usable price, so + # the caller carries the last real one forward. + assert parse_level_one(zero) == [(1786356933000, None, 2)] + + +def test_a_zero_price_with_no_trade_markers_is_dropped_entirely(): + assert parse_level_one({"content": [{"key": "/ES", "LAST_PRICE": 0}]}) == [] diff --git a/tests/test_store.py b/tests/test_store.py index 4db3f4e..18cca55 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -15,3 +15,36 @@ def test_store_replaces_forming_bar_and_bounds_history(): assert [value.t for value in store.get(Timeframe.M1)] == [120, 180] assert store.get(Timeframe.M1, 1)[0].t == 180 + + +def test_a_closed_bar_replaces_its_bucket_behind_the_tail(): + # Ticks open the next minute before the exchange's own bar for the previous + # one arrives. Matching only the tail dropped it, leaving the tick-built + # approximation — with its partial volume — in place permanently. + from app.bars.models import Bar, Timeframe + from app.bars.store import InMemoryBarStore + + store = InMemoryBarStore(100) + common = dict(tf=Timeframe.M1, symbol="/ES", source="schwab") + store.put(Bar(t=60, o=1, h=2, l=1, c=2, v=5, closed=False, **common)) + store.put(Bar(t=120, o=2, h=3, l=2, c=3, v=1, closed=False, **common)) + # The authoritative bar for the first minute, arriving late. + store.put(Bar(t=60, o=1, h=9, l=1, c=4, v=400, closed=True, **common)) + + held = store.get(Timeframe.M1) + assert [b.t for b in held] == [60, 120] + assert held[0].closed is True + assert held[0].v == 400, "the exchange's volume must win over the tick estimate" + + +def test_a_tick_cannot_overwrite_a_settled_bar(): + from app.bars.models import Bar, Timeframe + from app.bars.store import InMemoryBarStore + + store = InMemoryBarStore(100) + common = dict(tf=Timeframe.M1, symbol="/ES", source="schwab") + store.put(Bar(t=60, o=1, h=9, l=1, c=4, v=400, closed=True, **common)) + store.put(Bar(t=60, o=1, h=2, l=1, c=2, v=5, closed=False, **common)) + + held = store.get(Timeframe.M1)[0] + assert held.closed is True and held.v == 400