From 374e255c952d7019a49191acbdc83ad1d3b12824 Mon Sep 17 00:00:00 2001 From: Chris Amow Date: Mon, 10 Aug 2026 07:58:18 -0500 Subject: [PATCH] Keep the volume from trades that print at an unchanged price MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The candle still paused for ten to twenty seconds at a time. Instrumenting the raw Level 1 stream settled why: 87 messages in 90 seconds, only 33 carrying LAST_PRICE. Most of the remainder is bid and ask movement, correctly ignored, but a seventh carry LAST_SIZE, TRADE_TIME_MILLIS and TOTAL_VOLUME with no LAST_PRICE — trades that printed at the price of the one before, so the field did not change and Level 1 did not resend it. Requiring LAST_PRICE discarded those trades and their volume with them. parse_level_one now recognises size-plus-trade-time as a trade and returns a null price, which stream() fills from the forming bar. A quote carrying neither a price nor any trade field is still skipped: a bid is not a trade and must not extend a candle's high or low. Measured on the live feed: median gap between updates 3.1s to 2.0s, worst gap 21.5s to 8.1s, roughly 9 updates a minute to 22, and bar volume climbs within the minute instead of standing still. The pauses that remain are the market rather than the pipe. Thin pre-open tape goes seconds without a price-changing trade and then moves several ticks at once, which is what a gap up after a quiet spell is. Co-Authored-By: Claude Opus 5 --- app/market/schwab.py | 33 +++++++++++++++++++++++++-------- docs/IMPLEMENTATION_PLAN.md | 24 ++++++++++++++++++++++++ tests/test_schwab_source.py | 16 ++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/app/market/schwab.py b/app/market/schwab.py index 5c96593..27ba965 100644 --- a/app/market/schwab.py +++ b/app/market/schwab.py @@ -39,25 +39,33 @@ FIELD_LAST_SIZE = "LAST_SIZE" FIELD_TRADE_TIME = "TRADE_TIME_MILLIS" -def parse_level_one(message: dict) -> list[tuple[int, float, int]]: +def parse_level_one(message: dict) -> list[tuple[int, float | None, int]]: """Turn one LEVEL_ONE_FUTURES message into (trade time ms, price, size). - Level 1 messages are partial: a quote that moves only the bid carries no - LAST_PRICE at all. Those are skipped rather than carried forward, because a - bid tick is not a trade and must not extend a candle's high or low. + Level 1 messages are partial — only changed fields are sent — which makes + "is this a trade?" a question about several fields rather than one: + + - A quote moving only the bid or ask carries no trade field at all. Skipped: + a bid is not a trade and must not extend a candle's high or low. + - A trade at the *same price* as the one before carries LAST_SIZE and + TRADE_TIME_MILLIS but no LAST_PRICE, because the price did not change. + Measured live, that is a fifth of all trades. Dropping them lost their + volume, so the price is returned as None for the caller to carry forward. """ - ticks: list[tuple[int, float, int]] = [] + ticks: list[tuple[int, float | None, int]] = [] for content in message.get("content") or []: price = content.get(FIELD_LAST_PRICE) - if price is None: + size = content.get(FIELD_LAST_SIZE) + traded_at = content.get(FIELD_TRADE_TIME) + if price is None and (size is None or traded_at is None): continue - millis = content.get(FIELD_TRADE_TIME) + millis = traded_at if millis is None: # No trade stamp on this update; the wall clock is close enough to # bucket it, and being one minute out at a boundary is corrected by # the authoritative CHART_FUTURES bar moments later. millis = int(time.time() * 1000) - ticks.append((int(millis), float(price), int(content.get(FIELD_LAST_SIZE) or 0))) + ticks.append((int(millis), None if price is None else float(price), int(size or 0))) return ticks @@ -165,6 +173,7 @@ class SchwabSource: forming: Bar | None = None last_closed_t = 0 last_emit = 0.0 + last_price: float | None = None pump = asyncio.create_task(self._pump(stream_client), name="schwab-stream-pump") try: while True: @@ -187,6 +196,14 @@ class SchwabSource: yield bar continue for millis, price, size in parse_level_one(message): + if price is None: + # A same-price trade. Carry the last price forward so + # its volume still lands; without a price to stand on + # there is no bar to add it to. + price = forming.c if forming is not None else last_price + if price is None: + continue + last_price = price minute = millis // 60000 * 60 # A tick for a minute already closed by CHART_FUTURES would # otherwise overwrite an authoritative bar with a partial. diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 725230b..0544185 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -1339,3 +1339,27 @@ behind the last one has nothing to contribute. Verified: 36 rapid timeframe switches under a live tick feed produce zero errors, and calling `candles.update()` directly with a stale bar still throws while the guarded `updateBar()` does not. + +**Same-price trades were being dropped.** The candle still paused for 10–20 +seconds at a time after Level 1 went in. Instrumenting the raw stream settled +it: 87 messages in 90 seconds, only 33 carrying `LAST_PRICE`. Most of the rest +are pure bid/ask movement and correctly ignored — but a seventh of them look +like this: + +``` +['ASK_SIZE','ASK_TIME_MILLIS','BID_SIZE','BID_TIME_MILLIS', + 'LAST_SIZE','QUOTE_TIME_MILLIS','TOTAL_VOLUME','TRADE_TIME_MILLIS','key'] +``` + +Trade time, trade size, cumulative volume — and no `LAST_PRICE`, because Level 1 +sends only *changed* fields and the trade printed at the price of the one +before. Requiring `LAST_PRICE` threw those away along with their volume. +`parse_level_one` now treats size-plus-trade-time as a trade and returns a null +price for the caller to carry forward. Measured on the live feed: median gap +3.1s → 2.0s, worst 21.5s → 8.1s, and bar volume climbs within the minute instead +of standing still. + +Worth recording for the next person who reads a gap as a bug: the remaining +pauses are the market, not the pipe. In thin pre-open tape /ES genuinely goes +seconds without a price-changing trade, and then moves several ticks at once — +which is what a "gap up" after a quiet spell actually is. diff --git a/tests/test_schwab_source.py b/tests/test_schwab_source.py index 283377d..9dbce2c 100644 --- a/tests/test_schwab_source.py +++ b/tests/test_schwab_source.py @@ -251,3 +251,19 @@ def test_a_tick_for_an_already_closed_minute_is_ignored(): seen = asyncio.run(asyncio.wait_for(two_bars(), timeout=10)) assert [bar.closed for bar in seen] == [True] + + +def test_a_same_price_trade_keeps_its_volume(): + # Level 1 sends only changed fields, so a trade at the price of the one + # before carries size and trade time but no LAST_PRICE. Measured live at a + # fifth of all trades; dropping them lost that volume from the bar. + same_price = { + "content": [ + {"key": "/ES", "LAST_SIZE": 4, "TRADE_TIME_MILLIS": 1786356931000, "TOTAL_VOLUME": 9} + ] + } + assert parse_level_one(same_price) == [(1786356931000, None, 4)] + + +def test_a_quote_with_neither_price_nor_trade_is_still_skipped(): + assert parse_level_one({"content": [{"key": "/ES", "BID_SIZE": 12}]}) == []