diff --git a/app/market/yahoo.py b/app/market/yahoo.py index 3b9641f..62051b6 100644 --- a/app/market/yahoo.py +++ b/app/market/yahoo.py @@ -3,8 +3,10 @@ from collections.abc import AsyncIterator from typing import Any import httpx +from dataclasses import replace from app.bars.models import Bar, Timeframe +from app.bars.session import bucket_start YAHOO_CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" MAX_1M_WINDOW_SECONDS = 8 * 24 * 60 * 60 @@ -28,6 +30,12 @@ def parse_chart(payload: dict[str, Any], tf: Timeframe, source: str = "yahoo") - t, open_, high, low, close, volume = values if any(value is None for value in (t, open_, high, low, close)): continue + # Yahoo stamps the in-progress candle with the moment of the request, + # not the start of its bucket. Emitted verbatim, every poll produced a + # new "1m" bar a few seconds after the last — 04:38:11, 04:38:50, + # 04:39:15 — instead of revising the current minute. Bucketing makes the + # partial candle land on its own minute, where the store replaces it. + t = bucket_start(int(t), tf) bars.append( Bar( tf=tf, @@ -110,8 +118,17 @@ class YahooSource: last_emitted = -1 while True: bars = await self.history(symbol, Timeframe.M1, range_="1d") - for bar in bars: - if bar.t > last_emitted: - yield bar + for index, bar in enumerate(bars): + # The final candle is still forming. Marked unclosed it revises + # the last bar and the live price without entering the + # aggregator, which would otherwise add its volume to every + # higher timeframe again on every poll. last_emitted tracks the + # newest *settled* bar, so the forming minute is re-sent each + # poll and finally sent once more as closed. + forming = index == len(bars) - 1 + if bar.t < last_emitted or (bar.t == last_emitted and not forming): + continue + yield replace(bar, closed=not forming) + if not forming: last_emitted = bar.t await asyncio.sleep(self.poll_seconds) diff --git a/docs/implementation.md b/docs/implementation.md index 5a76114..52865cf 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -766,3 +766,50 @@ display-media and upload-completion boundaries. These are explicit `node:test` skips with reasons, not deleted coverage. Re-enable each case only after its stated external dependency is removed and repeated full suite runs remain green. + +### 2026-08-11 — a time axis past the last bar, and the Yahoo bar it exposed + +Trendlines project into the whitespace right of the last candle, but the axis +stopped at the newest bar — so two lines could be seen converging with no way to +tell when. Lightweight Charts only labels times that exist on its scale, so the +fix is whitespace data: `{ time }` points with no value, which extend the scale +and draw nothing. `timeToCoordinate` now answers out there too, which the +projection layer wants anyway. + +Future times repeat the most recent bar interval, matching what `timeAtIndex` +already does for the projections. That drifts across the daily halt and the +weekend, because futures do not trade through them. Consistency with the +projection matters more than being right in the abstract — an axis disagreeing +with the line drawn above it would be worse — but session-accurate projection +needs the server's session rules, which the client does not have. + +**The interesting part was what this exposed.** Padding that should have been +five bars measured as sixty-seven, because both the padding and the future +slots took the interval from the gap between the last two bars. That gap was two +seconds on a one-minute chart, so `barInterval` now takes a median over recent +bars, which ignores a ragged tail. + +Chasing *why* the last gap was two seconds found a real data bug, live in +production, which is still on Yahoo. Yahoo stamps the in-progress candle with +the moment of the request rather than its bucket start, and the poller emitted +anything newer than the last thing it sent — so every poll appended a new "1m" +bar a few seconds after the previous one: + +``` +04:38:11 aligned=False +04:38:50 aligned=False +04:39:00 aligned=True <- the only real bar +04:39:15 aligned=False +gaps: 10, 15, 30, 15, 2, 26, 20, 12, 8, 17 seconds +``` + +Timestamps are bucketed on parse now, and the final candle is emitted unclosed +so it revises the current minute instead of entering the aggregator — which +would otherwise add its volume to every higher timeframe on every poll. +`last_emitted` tracks the newest settled bar, so the forming minute is re-sent +each poll and once more when it closes. Live check after the fix: eight +consecutive bars, all aligned, all sixty seconds apart. + +Worth generalising: a measurement that looks absurd is worth chasing rather than +clamping. The absurd number here was "67 bars of padding", and the bug behind it +had nothing to do with padding. diff --git a/static/chart.js b/static/chart.js index 1770a7b..07cb743 100644 --- a/static/chart.js +++ b/static/chart.js @@ -48,6 +48,7 @@ class ConfluenceChart { this.toolUpListener = null; this.pendingView = null; this.overlayLayer = null; + this.futureSpace = null; this.snapDot = null; this.snapLabel = null; this.snapLeader = null; @@ -69,6 +70,30 @@ class ConfluenceChart { // rather than a click. Wide enough to survive a twitch on a deliberate click. static DRAG_THRESHOLD = 12; + // How far past the last bar the axis is extended. Enough to date a + // convergence a few sessions out without dominating a zoomed-out view. + static FUTURE_SLOTS = 180; + + /** + * The timeframe's spacing, measured rather than assumed. + * + * Taking the gap between the final two bars looks equivalent and is not: the + * newest bar is often partial or irregularly stamped — a live one arrived at + * :02 past the minute — which made that gap two seconds on a one-minute + * chart. Everything derived from it was then wrong by a factor of thirty. A + * median over recent bars ignores both the ragged tail and session gaps. + */ + static barInterval(bars) { + const deltas = []; + for (let i = Math.max(1, bars.length - 30); i < bars.length; i += 1) { + const delta = bars[i].t - bars[i - 1].t; + if (delta > 0) deltas.push(delta); + } + if (!deltas.length) return 0; + deltas.sort((a, b) => a - b); + return deltas[Math.floor(deltas.length / 2)]; + } + /** * Whether to report snap geometry to the server. * @@ -170,6 +195,18 @@ class ConfluenceChart { // which is where the volume draws and where a cursor tracing swing lows // naturally sits — far from any actual price. Give most of it back. this.chart.priceScale('right').applyOptions({ scaleMargins: { top: 0.12, bottom: 0.04 } }); + // Whitespace to the right of the last bar. Lightweight Charts only labels + // times that exist on the scale, so the axis under the projection area was + // blank — you could watch two trendlines converge and not know when. These + // points carry no value and draw nothing; they exist so the axis has + // something to label and so timeToCoordinate answers out there. + this.futureSpace = this.chart.addSeries(LightweightCharts.LineSeries, { + priceScaleId: 'right', + lastValueVisible: false, + priceLineVisible: false, + crosshairMarkerVisible: false, + autoscaleInfoProvider: () => null, + }); this.resizeObserver = new ResizeObserver(() => { this.chart.applyOptions({ width: el.clientWidth, height: el.clientHeight }); requestAnimationFrame(() => { @@ -343,6 +380,7 @@ class ConfluenceChart { const candleData = bars.map(this.toCandle); this.candles.setData(candleData); this.volume.setData(bars.map(ConfluenceChart.toVolume)); + this.futureSpace.setData(this.futureTimes(bars)); // Anchored by time, not by logical index. A logical index addresses the // chart's *shared* scale — the union of every series' time points — not // this array. The daily MAs land straight after with hundreds of points @@ -351,8 +389,8 @@ class ConfluenceChart { // A time range names the instant, so later series cannot move it. if (bars.length) { const last = bars[bars.length - 1]; - // Keep the old five bars of right-hand breathing room, in seconds. - const step = bars.length > 1 ? last.t - bars[bars.length - 2].t : 60; + // Five bars of right-hand breathing room, in seconds. + const step = ConfluenceChart.barInterval(bars) || 60; this.pendingView = { from: bars[Math.max(0, bars.length - 160)].t, to: last.t + step * 5, @@ -366,6 +404,30 @@ class ConfluenceChart { this.renderTrendlineProjections(); } + /** + * Empty slots past the last bar, so the axis can label the projection area. + * + * Times are extrapolated by repeating the most recent bar interval, which is + * exactly what `timeAtIndex` does for the projections themselves. That is + * approximate across a session break — futures do not trade through the + * daily halt or the weekend, so a label far to the right drifts from the + * real clock. Being consistent with the projection matters more than being + * right in the abstract: an axis that disagreed with the line drawn above it + * would be worse than one that is uniformly optimistic. Session-accurate + * projection needs the server's session rules, which are not on the client. + */ + futureTimes(bars) { + if (bars.length < 2) return []; + const last = bars[bars.length - 1]; + const step = ConfluenceChart.barInterval(bars); + if (step <= 0) return []; + const slots = []; + for (let index = 1; index <= ConfluenceChart.FUTURE_SLOTS; index += 1) { + slots.push({ time: last.t + index * step }); + } + return slots; + } + updateBar(bar) { // update() throws on anything older than the series' last point, which // takes the whole app down rather than dropping one stale bar. Ticks made diff --git a/tests/test_yahoo.py b/tests/test_yahoo.py index c268df4..5042974 100644 --- a/tests/test_yahoo.py +++ b/tests/test_yahoo.py @@ -58,3 +58,27 @@ async def test_thirty_minute_history_uses_yahoos_native_interval(): assert calls[0].url.params["interval"] == "30m" assert calls[0].url.params["range"] == "60d" assert bars and all(bar.tf is Timeframe.M30 for bar in bars) + + +def test_the_forming_candle_is_bucketed_not_stamped_at_request_time(): + # Yahoo stamps the in-progress candle with the moment of the request. Taken + # verbatim, every poll appended a new "1m" bar seconds after the last — + # 04:38:11, 04:38:50, 04:39:15 — instead of revising the current minute. + from app.bars.models import Timeframe + from app.market.yahoo import parse_chart + + payload = { + "chart": {"result": [{ + "timestamp": [1786500000, 1786500060, 1786500071], # last is mid-minute + "indicators": {"quote": [{ + "open": [1, 2, 3], "high": [1, 2, 3], + "low": [1, 2, 3], "close": [1, 2, 3], "volume": [1, 1, 1], + }]}, + "meta": {"symbol": "ES=F"}, + }]} + } + + times = [bar.t for bar in parse_chart(payload, Timeframe.M1)] + + assert all(t % 60 == 0 for t in times), f"unaligned bar times: {times}" + assert times[-1] == 1786500060, "the forming candle did not fold onto its own minute"