From 2e8e07ed177e374206c2ac7fcb3b78a19f87219f Mon Sep 17 00:00:00 2001 From: Kunthawat Greethong Date: Tue, 6 Jan 2026 12:16:53 +0700 Subject: [PATCH] refactor(oi): update scraper for new QuikStrike website structure - Replace direct product URL navigation with fixed heatmap URL and UI product selection - Implement cookie validation with automatic session cleanup - Update login flow to use SSO authentication and new form selectors - Improve data extraction with iframe context and better table parsing - Add multiple fallback selectors for gold price scraping - Enhance error handling, logging, and timeout management --- .DS_Store | Bin 0 -> 6148 bytes OI_OrderFlow_Absorption_XAUUSD.mq5 | 1238 ++++++++++++++++++++++++++++ oi_scraper/.env.example | 11 +- oi_scraper/README.md | 1 + oi_scraper/main.py | 244 ++++-- 5 files changed, 1411 insertions(+), 83 deletions(-) create mode 100644 .DS_Store create mode 100644 OI_OrderFlow_Absorption_XAUUSD.mq5 diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..cfdf775b470e9d9d0f297c877a1c757b13d136d9 GIT binary patch literal 6148 zcmeHKJ5EC}5S)b+5i}_&eFbh{Md1V-fKNh!6zL%%`d4wT94)h-g6Ih;MbbpG(t7Om zjxA5|_ALNg{%jtB1%Nr-5x+i6&G+3Wc2f}}(s{-j-+03pK8UB8bMG+V6EE2PLuSI;@%xtDCJl6pPz=ev5QiPt+&{q` +#include +#include +#include +#include +#include +#include +#include + +enum ENUM_MARKET_PHASE { + PHASE_BULLISH, + PHASE_BEARISH, + PHASE_SIDEWAYS, + PHASE_NEUTRAL +}; + +enum ENUM_ABSORPTION_STATE { + ABSORPTION_NONE, + ABSORPTION_BUY, + ABSORPTION_SELL +}; + +enum ENUM_OI_SOURCE { + OI_SOURCE_MANUAL, + OI_SOURCE_CSV_FILE, + OI_SOURCE_AUTO_SYNC +}; + +input group "=== OI & DELTA SETTINGS ===" +input ENUM_OI_SOURCE InpOISource = OI_SOURCE_MANUAL; +input string InpOICsvPath = "\\Files\\oi_data.csv"; +input double InpManualFuturePrice = 0.0; + +input double InpCallStrike1 = 0.0; +input double InpCallStrike2 = 0.0; +input double InpCallStrike3 = 0.0; +input double InpPutStrike1 = 0.0; +input double InpPutStrike2 = 0.0; +input double InpPutStrike3 = 0.0; + +input group "=== ORDER FLOW & ABSORPTION ===" +input bool InpUseAbsorptionFilter = true; +input int InpAbsorptionBars = 3; +input int InpOIZonePoints = 100; +input double InpMinVolumeMultiplier = 1.5; +input int InpVolumeEmaPeriod = 20; +input int InpMaxPriceDriftPoints = 50; +input int InpSLBufferPoints = 200; + +input group "=== TRADING SETTINGS ===" +input double InpLotSize = 0.1; +input bool InpUseMoneyManagement = true; +input double InpRiskPercent = 1.0; +input bool InpUseStopLoss = true; +input bool InpUseTakeProfit = true; +input int InpStopLossPoints = 300; +input int InpTakeProfitPoints = 500; +input int InpMaxSpread = 200; +input int InpMaxSlippage = 10; +input int InpMagicNumber = 202501; + +input group "=== MARKET FILTERS ===" +input bool InpUseMarketPhaseFilter = true; +input ENUM_TIMEFRAMES InpTrendTF = PERIOD_H4; +input int InpTrendMAPeriod1 = 50; +input int InpTrendMAPeriod2 = 200; +input bool InpUseATRFilter = true; +input double InpMaxATRPercent = 2.0; +input bool InpUseSessionFilter = true; +input int InpSessionStartHour = 8; +input int InpSessionEndHour = 22; + +input group "=== RISK MANAGEMENT ===" +input double InpMaxDailyLossPercent = 3.0; +input double InpMaxDailyProfitPercent = 5.0; +input int InpMaxConsecutiveLosses = 3; +input bool InpDisableAfterMaxLoss = true; +input double InpEquityProtectionPercent = 10.0; +input bool InpCloseAllOnReverse = false; +input int InpMaxDailyTrades = 10; + +input group "=== ADVANCED FEATURES ===" +input bool InpUseTrailingStop = false; +input int InpTrailingStartPoints = 150; +input int InpTrailingStepPoints = 50; +input bool InpUseSoftStopLoss = true; +input bool InpUseAutoRecovery = true; +input bool InpEnableDashboard = true; + +CTrade Trade; +CSymbolInfo SymbolInfo; +CAccountInfo AccountInfo; +CPositionInfo PositionInfo; +COrderInfo OrderInfo; +CHistoryOrderInfo HistoryOrderInfo; + +double FuturePrice = 0.0; +double SpotPrice = 0.0; + +double CallLevels[3]; +double PutLevels[3]; + +double DynamicFuturePrice = 0.0; +double DynamicCallStrike1 = 0.0; +double DynamicCallStrike2 = 0.0; +double DynamicCallStrike3 = 0.0; +double DynamicPutStrike1 = 0.0; +double DynamicPutStrike2 = 0.0; +double DynamicPutStrike3 = 0.0; + +double LevelUpper1 = 0.0; +double LevelUpper2 = 0.0; +double LevelMid = 0.0; +double LevelLower1 = 0.0; +double LevelLower2 = 0.0; + +int DailyTradeCount = 0; +double DailyPnL = 0.0; +double DailyProfit = 0.0; +double DailyLoss = 0.0; +int ConsecutiveLosses = 0; +datetime LastTradeTime = 0; +datetime LastResetDate = 0; + +bool TradingEnabled = true; +double EquityHigh = 0.0; +double EquityLow = 0.0; + +int ATRHandle = INVALID_HANDLE; +int MAFastHandle = INVALID_HANDLE; +int MASlowHandle = INVALID_HANDLE; +int RSIMainHandle = INVALID_HANDLE; + +long chart_id = 0; +int DashboardSubWindow = -1; +color PanelColor = C'30,30,30'; +color TextColor = clrWhite; +color ProfitColor = clrLime; +color LossColor = clrRed; +color WarningColor = clrOrange; + +int ControlPanelX = 10; +int ControlPanelY = 210; + +double ManualFuturePriceValue = 0.0; +double CallStrike1Value = 0.0; +double CallStrike2Value = 0.0; +double CallStrike3Value = 0.0; +double PutStrike1Value = 0.0; +double PutStrike2Value = 0.0; +double PutStrike3Value = 0.0; + +string ControlPanelObjects[] = { + "CP_TopPanel", + "CP_CloseAll", + "CP_OIDataPanel", + "CP_FuturePrice", + "CP_CallStrike1", + "CP_CallStrike2", + "CP_CallStrike3", + "CP_PutStrike1", + "CP_PutStrike2", + "CP_PutStrike3", + "CP_UpdateOI" +}; + +string DrawnLines[]; +int MaxLines = 10; + +double OrderFlowDeltaPercent = 0.0; +double VolumeEmaValue = 0.0; +double PriceDrift = 0.0; +ENUM_ABSORPTION_STATE CurrentAbsorptionState = ABSORPTION_NONE; + +datetime LastM1BarTime = 0; +datetime LastDashboardUpdate = 0; + +int CurrentBarUpTicks = 0; +int CurrentBarDownTicks = 0; +int CurrentBarVolume = 0; +double CurrentBarHigh = 0.0; +double CurrentBarLow = 0.0; +double LastPrice = 0.0; + +int OnInit() { + Trade.SetExpertMagicNumber(InpMagicNumber); + Trade.SetDeviationInPoints(InpMaxSlippage); + Trade.SetTypeFilling(ORDER_FILLING_IOC); + + SymbolInfo.Name(_Symbol); + SymbolInfo.RefreshRates(); + + DynamicFuturePrice = InpManualFuturePrice; + DynamicCallStrike1 = InpCallStrike1; + DynamicCallStrike2 = InpCallStrike2; + DynamicCallStrike3 = InpCallStrike3; + DynamicPutStrike1 = InpPutStrike1; + DynamicPutStrike2 = InpPutStrike2; + DynamicPutStrike3 = InpPutStrike3; + + InitializeOILevels(); + InitializeKeyLevels(); + + if(!InitializeIndicators()) { + Print("Error initializing indicators"); + return INIT_FAILED; + } + + EquityHigh = AccountInfo.Equity(); + EquityLow = AccountInfo.Equity(); + + if(InpEnableDashboard) { + chart_id = ChartID(); + CreateDashboard(); + CreateControlPanel(); + } + + if(InpUseAutoRecovery) { + CheckExistingPositions(); + } + + EventSetTimer(1); + + ArrayResize(DrawnLines, MaxLines); + for(int i = 0; i < MaxLines; i++) { + DrawnLines[i] = ""; + } + + LastPrice = SymbolInfo.Bid(); + CurrentBarHigh = LastPrice; + CurrentBarLow = LastPrice; + LastM1BarTime = iTime(_Symbol, PERIOD_M1, 0); + + Print("EA Initialized Successfully: Order Flow Absorption Strategy"); + Print("Symbol: ", _Symbol); + + return INIT_SUCCEEDED; +} + +void OnDeinit(const int reason) { + if(InpEnableDashboard) { + ObjectsDeleteAll(chart_id, 0, -1); + } + + if(ATRHandle != INVALID_HANDLE) IndicatorRelease(ATRHandle); + if(MAFastHandle != INVALID_HANDLE) IndicatorRelease(MAFastHandle); + if(MASlowHandle != INVALID_HANDLE) IndicatorRelease(MASlowHandle); + if(RSIMainHandle != INVALID_HANDLE) IndicatorRelease(RSIMainHandle); + + EventKillTimer(); + Print("EA Deinitialized"); +} + +void OnTick() { + if(Bars(_Symbol, _Period) < 100) + return; + + UpdateMarketData(); + CountTicks(); + UpdatePriceDrift(); + + datetime currentM1BarTime = iTime(_Symbol, PERIOD_M1, 0); + + if(currentM1BarTime != LastM1BarTime) { + OnNewM1Bar(currentM1BarTime); + } + + if(!CheckGlobalConditions()) + return; + + CheckTradingSignals(); + ManagePositions(); + + if(InpEnableDashboard && TimeCurrent() - LastDashboardUpdate >= 1) { + UpdateDashboard(); + UpdateControlPanel(); + LastDashboardUpdate = TimeCurrent(); + } +} + +void OnTimer() { + if(InpEnableDashboard && TimeCurrent() - LastDashboardUpdate >= 1) { + UpdateDashboard(); + UpdateControlPanel(); + LastDashboardUpdate = TimeCurrent(); + } + + CheckDailyReset(); +} + +void OnChartEvent(const int id, const long &lparam, const double &dparam, const string &sparam) { + if(id == CHARTEVENT_OBJECT_CLICK) { + HandleControlPanelClick(sparam); + } + + if(id == CHARTEVENT_OBJECT_ENDEDIT) { + UpdateInputValues(); + Print("Edit field updated: ", sparam); + } +} + +void CountTicks() { + double currentPrice = SymbolInfo.Bid(); + + if(LastPrice == 0) { + LastPrice = currentPrice; + CurrentBarHigh = currentPrice; + CurrentBarLow = currentPrice; + return; + } + + if(currentPrice > LastPrice) { + CurrentBarUpTicks++; + } else if(currentPrice < LastPrice) { + CurrentBarDownTicks++; + } + + CurrentBarVolume++; + LastPrice = currentPrice; +} + +void UpdatePriceDrift() { + CurrentBarHigh = MathMax(CurrentBarHigh, SpotPrice); + CurrentBarLow = MathMin(CurrentBarLow, SpotPrice); + PriceDrift = CurrentBarHigh - CurrentBarLow; +} + +void OnNewM1Bar(datetime newBarTime) { + int totalTicks = CurrentBarUpTicks + CurrentBarDownTicks; + + if(totalTicks > 0) { + OrderFlowDeltaPercent = ((double)CurrentBarUpTicks - (double)CurrentBarDownTicks) / totalTicks * 100.0; + } else { + OrderFlowDeltaPercent = 0; + } + + CalculateVolumeEMAFromHistory(); + DetectAbsorptionFromHistory(); + + CurrentBarUpTicks = 0; + CurrentBarDownTicks = 0; + CurrentBarVolume = 0; + CurrentBarHigh = SpotPrice; + CurrentBarLow = SpotPrice; + LastM1BarTime = newBarTime; +} + +void CalculateVolumeEMAFromHistory() { + double volumeArray[]; + ArraySetAsSeries(volumeArray, true); + CopyVolume(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, volumeArray); + + double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1); + double sum = 0; + + for(int i = 0; i < InpVolumeEmaPeriod; i++) { + sum += volumeArray[i]; + } + double avgVolume = sum / InpVolumeEmaPeriod; + + if(VolumeEmaValue == 0) { + VolumeEmaValue = avgVolume; + } else { + VolumeEmaValue = emaAlpha * avgVolume + (1 - emaAlpha) * VolumeEmaValue; + } +} + +void DetectAbsorptionFromHistory() { + double volumeArray[]; + ArraySetAsSeries(volumeArray, true); + CopyVolume(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, volumeArray); + + double avgVolume = 0; + for(int i = 0; i < InpAbsorptionBars; i++) { + avgVolume += volumeArray[i]; + } + avgVolume /= InpAbsorptionBars; + + double volumeThreshold = avgVolume * InpMinVolumeMultiplier; + + int sellAbsorptionCount = 0; + int buyAbsorptionCount = 0; + + for(int i = 1; i <= InpAbsorptionBars; i++) { + double high = iHigh(_Symbol, PERIOD_M1, i); + double low = iLow(_Symbol, PERIOD_M1, i); + double close = iClose(_Symbol, PERIOD_M1, i); + double open = iOpen(_Symbol, PERIOD_M1, i); + int barVolume = (int)iVolume(_Symbol, PERIOD_M1, i); + + double barRange = high - low; + double barDrift = close - open; + + bool highVolume = barVolume > volumeThreshold; + bool lowDrift = barRange < InpMaxPriceDriftPoints * _Point; + + if(highVolume && lowDrift && barDrift < 0) { + sellAbsorptionCount++; + } + + if(highVolume && lowDrift && barDrift > 0) { + buyAbsorptionCount++; + } + } + + int requiredBars = MathCeil(InpAbsorptionBars / 2.0); + + if(sellAbsorptionCount >= requiredBars && IsPriceNearPutStrike()) { + CurrentAbsorptionState = ABSORPTION_SELL; + } else if(buyAbsorptionCount >= requiredBars && IsPriceNearCallStrike()) { + CurrentAbsorptionState = ABSORPTION_BUY; + } else { + CurrentAbsorptionState = ABSORPTION_NONE; + } +} + +void UpdateMarketData() { + SpotPrice = SymbolInfo.Bid(); + SymbolInfo.RefreshRates(); + + double csvFuturePrice = LoadFuturePriceFromCSV(); + if(csvFuturePrice > 0) { + FuturePrice = csvFuturePrice; + } else if(DynamicFuturePrice > 0) { + FuturePrice = DynamicFuturePrice; + } else if(InpManualFuturePrice > 0) { + FuturePrice = InpManualFuturePrice; + } else { + FuturePrice = SpotPrice; + } +} + +bool IsPriceNearPutStrike() { + double tolerance = InpOIZonePoints * _Point; + + for(int i = 0; i < 3; i++) { + double strike = (i == 0) ? DynamicPutStrike1 : (i == 1) ? DynamicPutStrike2 : DynamicPutStrike3; + if(strike > 0 && MathAbs(SpotPrice - strike) <= tolerance) { + return true; + } + } + return false; +} + +bool IsPriceNearCallStrike() { + double tolerance = InpOIZonePoints * _Point; + + for(int i = 0; i < 3; i++) { + double strike = (i == 0) ? DynamicCallStrike1 : (i == 1) ? DynamicCallStrike2 : DynamicCallStrike3; + if(strike > 0 && MathAbs(SpotPrice - strike) <= tolerance) { + return true; + } + } + return false; +} + +bool IsInMiddleOfRange() { + double tolerance = InpOIZonePoints * _Point * 2; + + for(int i = 0; i < 3; i++) { + double putStrike = (i == 0) ? DynamicPutStrike1 : (i == 1) ? DynamicPutStrike2 : DynamicPutStrike3; + double callStrike = (i == 0) ? DynamicCallStrike1 : (i == 1) ? DynamicCallStrike2 : DynamicCallStrike3; + + if(putStrike > 0 && MathAbs(SpotPrice - putStrike) <= tolerance) return false; + if(callStrike > 0 && MathAbs(SpotPrice - callStrike) <= tolerance) return false; + } + + return true; +} + +bool CheckGlobalConditions() { + if(!TradingEnabled) return false; + + if(!TerminalInfoInteger(TERMINAL_CONNECTED)) return false; + if(!TerminalInfoInteger(TERMINAL_TRADE_ALLOWED) || !MQLInfoInteger(MQL_TRADE_ALLOWED)) return false; + + long spread = SymbolInfo.Spread(); + if(spread > InpMaxSpread) return false; + + if(InpUseSessionFilter && !IsTradingSession()) return false; + + if(InpUseATRFilter && !CheckVolatilityFilter()) return false; + + if(!CheckDailyLimits()) { + TradingEnabled = false; + return false; + } + + if(CheckEquityProtection()) { + TradingEnabled = false; + return false; + } + + return true; +} + +bool IsTradingSession() { + MqlDateTime dt; + TimeToStruct(TimeCurrent(), dt); + return dt.hour >= InpSessionStartHour && dt.hour <= InpSessionEndHour; +} + +bool CheckVolatilityFilter() { + if(ATRHandle == INVALID_HANDLE) return true; + + double atrValues[]; + ArraySetAsSeries(atrValues, true); + if(CopyBuffer(ATRHandle, 0, 0, 1, atrValues) < 1) return true; + + double atrValue = atrValues[0]; + double atrPercent = (atrValue / SpotPrice) * 100.0; + + return atrPercent <= InpMaxATRPercent; +} + +bool CheckDailyLimits() { + if(DailyTradeCount >= InpMaxDailyTrades) return false; + + if(DailyLoss >= AccountInfo.Balance() * (InpMaxDailyLossPercent / 100.0)) return false; + if(DailyProfit >= AccountInfo.Balance() * (InpMaxDailyProfitPercent / 100.0)) return false; + + return true; +} + +bool CheckEquityProtection() { + double currentEquity = AccountInfo.Equity(); + + if(EquityHigh == 0) EquityHigh = currentEquity; + if(currentEquity > EquityHigh) EquityHigh = currentEquity; + + double dropPercent = (EquityHigh - currentEquity) / EquityHigh * 100.0; + + if(dropPercent >= InpEquityProtectionPercent) { + EquityLow = currentEquity; + return true; + } + + return false; +} + +void CheckDailyReset() { + datetime now = TimeCurrent(); + MqlDateTime dtNow, dtLast; + TimeToStruct(now, dtNow); + TimeToStruct(LastResetDate, dtLast); + + if(LastResetDate == 0 || dtLast.day_of_week != dtNow.day_of_week) { + if(LastResetDate != 0 && dtLast.day != dtNow.day) { + DailyTradeCount = 0; + DailyPnL = 0; + DailyProfit = 0; + DailyLoss = 0; + ConsecutiveLosses = 0; + LastResetDate = now; + TradingEnabled = true; + Print("Daily statistics reset"); + } else { + LastResetDate = now; + } + } +} + +void CheckTradingSignals() { + if(PositionsTotal() > 0) return; + + if(InpUseAbsorptionFilter && IsInMiddleOfRange()) return; + + ENUM_MARKET_PHASE marketPhase = GetMarketPhase(); + + if(CheckSellConditions(marketPhase)) { + ExecuteSellTrade(); + } else if(CheckBuyConditions(marketPhase)) { + ExecuteBuyTrade(); + } +} + +ENUM_MARKET_PHASE GetMarketPhase() { + if(!InpUseMarketPhaseFilter) return PHASE_NEUTRAL; + + double maFast[], maSlow[]; + ArraySetAsSeries(maFast, true); + ArraySetAsSeries(maSlow, true); + + if(CopyBuffer(MAFastHandle, 0, 0, 2, maFast) < 2) return PHASE_NEUTRAL; + if(CopyBuffer(MASlowHandle, 0, 0, 1, maSlow) < 1) return PHASE_NEUTRAL; + + double maFastCurrent = maFast[0]; + double maFastPrevious = maFast[1]; + double maSlowCurrent = maSlow[0]; + + double slope = maFastCurrent - maFastPrevious; + + if(maFastCurrent > maSlowCurrent && slope > 0) return PHASE_BULLISH; + else if(maFastCurrent < maSlowCurrent && slope < 0) return PHASE_BEARISH; + else if(MathAbs(slope) < 0.0001) return PHASE_SIDEWAYS; + else return PHASE_NEUTRAL; +} + +bool CheckSellConditions(ENUM_MARKET_PHASE marketPhase) { + if(!IsPriceNearCallStrike()) return false; + + if(InpUseAbsorptionFilter) { + if(CurrentAbsorptionState != ABSORPTION_BUY) return false; + } + + if(!CheckOverbought()) return false; + if(!IsAtResistance()) return false; + + return true; +} + +bool CheckBuyConditions(ENUM_MARKET_PHASE marketPhase) { + if(!IsPriceNearPutStrike()) return false; + + if(InpUseAbsorptionFilter) { + if(CurrentAbsorptionState != ABSORPTION_SELL) return false; + } + + if(!CheckOversold()) return false; + if(!IsAtSupport()) return false; + + return true; +} + +bool CheckOverbought() { + if(RSIMainHandle != INVALID_HANDLE) { + double rsiValues[]; + ArraySetAsSeries(rsiValues, true); + if(CopyBuffer(RSIMainHandle, 0, 0, 1, rsiValues) >= 1) { + if(rsiValues[0] < 70) return false; + } + } + return true; +} + +bool CheckOversold() { + if(RSIMainHandle != INVALID_HANDLE) { + double rsiValues[]; + ArraySetAsSeries(rsiValues, true); + if(CopyBuffer(RSIMainHandle, 0, 0, 1, rsiValues) >= 1) { + if(rsiValues[0] > 30) return false; + } + } + return true; +} + +bool IsAtResistance() { + double tolerance = 100 * _Point; + + if(MathAbs(SpotPrice - LevelUpper1) <= tolerance || + MathAbs(SpotPrice - LevelUpper2) <= tolerance) { + return true; + } + + return false; +} + +bool IsAtSupport() { + double tolerance = 100 * _Point; + + if(MathAbs(SpotPrice - LevelLower1) <= tolerance || + MathAbs(SpotPrice - LevelLower2) <= tolerance) { + return true; + } + + return false; +} + +void ExecuteBuyTrade() { + double lotSize = CalculateLotSize(ORDER_TYPE_BUY); + if(lotSize <= 0) return; + + double sl = 0, tp = 0; + double nearestPutStrike = GetNearestPutStrike(); + + if(InpUseStopLoss && nearestPutStrike > 0) { + sl = nearestPutStrike - InpSLBufferPoints * _Point; + } else if(InpUseStopLoss) { + sl = SpotPrice - InpStopLossPoints * _Point; + } + + if(InpUseTakeProfit) { + double nearestCallStrike = GetNearestCallStrike(); + if(nearestCallStrike > 0) { + tp = nearestCallStrike; + } else { + tp = SpotPrice + InpTakeProfitPoints * _Point; + } + } + + sl = NormalizeDouble(sl, _Digits); + tp = NormalizeDouble(tp, _Digits); + + string comment = "OF_ABSORPTION_BUY"; + + if(Trade.Buy(lotSize, _Symbol, SpotPrice, sl, tp, comment)) { + Print("Buy order executed. Lot: ", lotSize, " SL: ", sl, " TP: ", tp); + DailyTradeCount++; + LastTradeTime = TimeCurrent(); + } else { + Print("Buy order failed: ", Trade.ResultRetcodeDescription()); + } +} + +void ExecuteSellTrade() { + double lotSize = CalculateLotSize(ORDER_TYPE_SELL); + if(lotSize <= 0) return; + + double sl = 0, tp = 0; + double nearestCallStrike = GetNearestCallStrike(); + + if(InpUseStopLoss && nearestCallStrike > 0) { + sl = nearestCallStrike + InpSLBufferPoints * _Point; + } else if(InpUseStopLoss) { + sl = SpotPrice + InpStopLossPoints * _Point; + } + + if(InpUseTakeProfit) { + double nearestPutStrike = GetNearestPutStrike(); + if(nearestPutStrike > 0) { + tp = nearestPutStrike; + } else { + tp = SpotPrice - InpTakeProfitPoints * _Point; + } + } + + sl = NormalizeDouble(sl, _Digits); + tp = NormalizeDouble(tp, _Digits); + + string comment = "OF_ABSORPTION_SELL"; + + if(Trade.Sell(lotSize, _Symbol, SpotPrice, sl, tp, comment)) { + Print("Sell order executed. Lot: ", lotSize, " SL: ", sl, " TP: ", tp); + DailyTradeCount++; + LastTradeTime = TimeCurrent(); + } else { + Print("Sell order failed: ", Trade.ResultRetcodeDescription()); + } +} + +double GetNearestPutStrike() { + double nearest = 0; + double minDistance = DBL_MAX; + + double strikes[3] = {DynamicPutStrike1, DynamicPutStrike2, DynamicPutStrike3}; + + for(int i = 0; i < 3; i++) { + if(strikes[i] > 0) { + double distance = MathAbs(SpotPrice - strikes[i]); + if(distance < minDistance) { + minDistance = distance; + nearest = strikes[i]; + } + } + } + + return nearest; +} + +double GetNearestCallStrike() { + double nearest = 0; + double minDistance = DBL_MAX; + + double strikes[3] = {DynamicCallStrike1, DynamicCallStrike2, DynamicCallStrike3}; + + for(int i = 0; i < 3; i++) { + if(strikes[i] > 0) { + double distance = MathAbs(SpotPrice - strikes[i]); + if(distance < minDistance) { + minDistance = distance; + nearest = strikes[i]; + } + } + } + + return nearest; +} + +double CalculateLotSize(ENUM_POSITION_TYPE tradeType) { + if(!InpUseMoneyManagement) { + return NormalizeLot(InpLotSize); + } + + double balance = AccountInfo.Balance(); + double riskAmount = balance * (InpRiskPercent / 100.0); + + double slDistance; + if(InpUseStopLoss) { + slDistance = InpStopLossPoints * _Point; + } else { + slDistance = 500 * _Point; + } + + double tickValue = SymbolInfo.TickValue(); + double tickSize = SymbolInfo.TickSize(); + + double lotSize = riskAmount / (slDistance * tickValue / tickSize); + + lotSize = NormalizeLot(lotSize); + + double maxLot = SymbolInfo.LotsMax(); + double minLot = SymbolInfo.LotsMin(); + + if(lotSize > maxLot) lotSize = maxLot; + if(lotSize < minLot) lotSize = 0; + + return lotSize; +} + +double NormalizeLot(double lot) { + double step = SymbolInfo.LotStep(); + return MathFloor(lot / step) * step; +} + +void ManagePositions() { + for(int i = PositionsTotal() - 1; i >= 0; i--) { + if(!PositionSelectByTicket(PositionGetTicket(i))) continue; + + string symbol = PositionGetString(POSITION_SYMBOL); + if(symbol != _Symbol) continue; + + int magic = (int)PositionGetInteger(POSITION_MAGIC); + if(magic != InpMagicNumber) continue; + + double currentProfit = PositionGetDouble(POSITION_PROFIT); + + DailyPnL += currentProfit; + if(currentProfit > 0) DailyProfit += currentProfit; + else DailyLoss += currentProfit; + + if(currentProfit < 0) { + ConsecutiveLosses++; + } else { + ConsecutiveLosses = 0; + } + + if(InpCloseAllOnReverse && ConsecutiveLosses >= InpMaxConsecutiveLosses) { + Trade.PositionClose(PositionGetTicket(i)); + TradingEnabled = false; + continue; + } + + if(InpUseTrailingStop) { + ManageTrailingStop(); + } + + if(InpUseSoftStopLoss && currentProfit > 0) { + double sl = PositionGetDouble(POSITION_SL); + double entryPrice = PositionGetDouble(POSITION_PRICE_OPEN); + double distanceToEntry = MathAbs(SpotPrice - entryPrice); + + if(distanceToEntry > InpTrailingStartPoints * _Point) { + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY) { + double newSl = SpotPrice - InpTrailingStepPoints * _Point; + if(newSl > sl) { + Trade.PositionModify(PositionGetTicket(i), newSl, PositionGetDouble(POSITION_TP)); + } + } else { + double newSl = SpotPrice + InpTrailingStepPoints * _Point; + if(newSl < sl || sl == 0) { + Trade.PositionModify(PositionGetTicket(i), newSl, PositionGetDouble(POSITION_TP)); + } + } + } + } + } +} + +void ManageTrailingStop() { + for(int i = PositionsTotal() - 1; i >= 0; i--) { + if(!PositionSelectByTicket(PositionGetTicket(i))) continue; + + string symbol = PositionGetString(POSITION_SYMBOL); + if(symbol != _Symbol) continue; + + int magic = (int)PositionGetInteger(POSITION_MAGIC); + if(magic != InpMagicNumber) continue; + + double currentSl = PositionGetDouble(POSITION_SL); + + if(PositionGetInteger(POSITION_TYPE) == POSITION_TYPE_BUY) { + double newSl = SpotPrice - InpTrailingStepPoints * _Point; + if(newSl > currentSl) { + Trade.PositionModify(PositionGetTicket(i), newSl, PositionGetDouble(POSITION_TP)); + } + } else { + double newSl = SpotPrice + InpTrailingStepPoints * _Point; + if(newSl < currentSl || currentSl == 0) { + Trade.PositionModify(PositionGetTicket(i), newSl, PositionGetDouble(POSITION_TP)); + } + } + } +} + +void CloseAllPositions() { + for(int i = PositionsTotal() - 1; i >= 0; i--) { + if(!PositionSelectByTicket(PositionGetTicket(i))) continue; + + string symbol = PositionGetString(POSITION_SYMBOL); + if(symbol != _Symbol) continue; + + int magic = (int)PositionGetInteger(POSITION_MAGIC); + if(magic == InpMagicNumber) { + Trade.PositionClose(PositionGetTicket(i)); + } + } +} + +void InitializeOILevels() { + CallLevels[0] = DynamicCallStrike1; + CallLevels[1] = DynamicCallStrike2; + CallLevels[2] = DynamicCallStrike3; + + PutLevels[0] = DynamicPutStrike1; + PutLevels[1] = DynamicPutStrike2; + PutLevels[2] = DynamicPutStrike3; +} + +void InitializeKeyLevels() { + double allLevels[]; + int count = 0; + + for(int i = 0; i < 3; i++) { + if(CallLevels[i] > 0) { + ArrayResize(allLevels, count + 1); + allLevels[count] = CallLevels[i]; + count++; + } + if(PutLevels[i] > 0) { + ArrayResize(allLevels, count + 1); + allLevels[count] = PutLevels[i]; + count++; + } + } + + if(count > 0) { + ArraySort(allLevels); + + LevelLower1 = allLevels[0]; + LevelLower2 = (count > 1) ? allLevels[1] : allLevels[0]; + LevelMid = (count > 2) ? allLevels[count/2] : allLevels[0]; + LevelUpper2 = (count > 1) ? allLevels[count-1] : allLevels[0]; + LevelUpper1 = (count > 2) ? allLevels[count-2] : allLevels[0]; + } +} + +bool InitializeIndicators() { + ATRHandle = iATR(_Symbol, PERIOD_H1, 14); + if(ATRHandle == INVALID_HANDLE) { + Print("Failed to create ATR indicator"); + return false; + } + + MAFastHandle = iMA(_Symbol, InpTrendTF, InpTrendMAPeriod1, 0, MODE_EMA, PRICE_CLOSE); + MASlowHandle = iMA(_Symbol, InpTrendTF, InpTrendMAPeriod2, 0, MODE_SMA, PRICE_CLOSE); + + if(MAFastHandle == INVALID_HANDLE || MASlowHandle == INVALID_HANDLE) { + Print("Failed to create MA indicators"); + return false; + } + + RSIMainHandle = iRSI(_Symbol, PERIOD_H1, 14, PRICE_CLOSE); + if(RSIMainHandle == INVALID_HANDLE) { + Print("Failed to create RSI indicator"); + return false; + } + + return true; +} + +double LoadFuturePriceFromCSV() { + string path = InpOICsvPath; + int filehandle = FileOpen(path, FILE_READ | FILE_CSV, ','); + + if(filehandle == INVALID_HANDLE) { + return 0.0; + } + + double futurePrice = 0.0; + + while(!FileIsEnding(filehandle)) { + string line = FileReadString(filehandle); + string parts[]; + int split = StringSplit(line, ',', parts); + + if(split >= 2) { + string dateStr = parts[0]; + double future = StringToDouble(parts[1]); + + if(future > 0) { + futurePrice = future; + break; + } + } + } + + FileClose(filehandle); + return futurePrice; +} + +void CheckExistingPositions() { + for(int i = 0; i < PositionsTotal(); i++) { + if(PositionSelectByTicket(PositionGetTicket(i))) { + string symbol = PositionGetString(POSITION_SYMBOL); + if(symbol == _Symbol) { + int magic = (int)PositionGetInteger(POSITION_MAGIC); + if(magic == InpMagicNumber) { + Print("Existing position found: ", PositionGetInteger(POSITION_TYPE)); + DailyTradeCount++; + } + } + } + } +} + +void CreateDashboard() { + int panelWidth = 280; + int panelHeight = 280; + + CreatePanel("DashboardPanel", 10, 10, panelWidth, panelHeight, C'25,25,35', BORDER_FLAT); + + CreateLabel("DB_Title", 20, 20, "ORDER FLOW ABSORPTION EA", clrYellow, 10); + + UpdateDashboard(); +} + +void UpdateDashboard() { + UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8); + UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8); + + string deltaText = DoubleToString(OrderFlowDeltaPercent, 1) + "%"; + color deltaColor = OrderFlowDeltaPercent > 0 ? clrLime : (OrderFlowDeltaPercent < 0 ? clrRed : clrGray); + UpdateLabel("DB_Delta", 20, 85, "Delta: " + deltaText, deltaColor, 8); + + string absorptionText = ""; + color absorptionColor = clrGray; + + switch(CurrentAbsorptionState) { + case ABSORPTION_BUY: + absorptionText = "BUY ABSORPTION"; + absorptionColor = clrLime; + break; + case ABSORPTION_SELL: + absorptionText = "SELL ABSORPTION"; + absorptionColor = clrRed; + break; + default: + absorptionText = "NONE"; + break; + } + + UpdateLabel("DB_Absorption", 20, 105, "Absorption: " + absorptionText, absorptionColor, 8); + + int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (int)Volume(0); + UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) + + " (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8); + + string driftText = DoubleToString(PriceDrift / _Point, 1) + " pts"; + color driftColor = PriceDrift < InpMaxPriceDriftPoints * _Point ? clrLime : clrOrange; + UpdateLabel("DB_PriceDrift", 20, 145, "Drift: " + driftText, driftColor, 8); + + UpdateLabel("DB_Trades", 20, 170, "Daily: " + IntegerToString(DailyTradeCount) + + "/" + IntegerToString(InpMaxDailyTrades), clrWhite, 8); + + UpdateLabel("DB_PnL", 20, 190, "Daily PnL: " + DoubleToString(DailyPnL, 2), + DailyPnL >= 0 ? clrLime : clrRed, 8); + + string tradingStatus = TradingEnabled ? "ENABLED" : "DISABLED"; + color statusColor = TradingEnabled ? clrLime : clrRed; + UpdateLabel("DB_Status", 20, 215, "Trading: " + tradingStatus, statusColor, 8); + + string nearZone = ""; + if(IsPriceNearPutStrike()) nearZone = "NEAR PUT"; + else if(IsPriceNearCallStrike()) nearZone = "NEAR CALL"; + else nearZone = "MIDDLE"; + + color zoneColor = (nearZone == "MIDDLE") ? clrOrange : clrCyan; + UpdateLabel("DB_Zone", 20, 235, "Zone: " + nearZone, zoneColor, 8); + + string ticksText = "Ticks: " + IntegerToString(CurrentBarUpTicks) + "/" + IntegerToString(CurrentBarDownTicks); + UpdateLabel("DB_Ticks", 20, 255, ticksText, clrWhite, 8); +} + +void CreateControlPanel() { + CreateControlPanelUI(); + UpdateControlPanelValues(); +} + +void CreateControlPanelUI() { + int panelWidth = 850; + int topPanelHeight = 50; + int oiPanelHeight = 120; + + CreatePanel("CP_TopPanel", ControlPanelX, ControlPanelY, panelWidth, topPanelHeight, C'45,45,45', BORDER_FLAT); + CreateButton("CP_CloseAll", ControlPanelX + 10, ControlPanelY + 10, 830, 30, "Close All Positions", clrRed, clrWhite); + + CreatePanel("CP_OIDataPanel", ControlPanelX, ControlPanelY + topPanelHeight + 10, panelWidth, oiPanelHeight, C'45,45,45', BORDER_FLAT); + + CreateLabel("CP_FuturePriceLabel", ControlPanelX + 10, ControlPanelY + topPanelHeight + 20, "Future Price:", clrYellow, 8); + CreateLabel("CP_CallStrikesLabel", ControlPanelX + 200, ControlPanelY + topPanelHeight + 20, "Call Strikes:", clrYellow, 8); + CreateLabel("CP_PutStrikesLabel", ControlPanelX + 200, ControlPanelY + topPanelHeight + 50, "Put Strikes:", clrYellow, 8); + + CreateEditField("CP_FuturePrice", ControlPanelX + 90, ControlPanelY + topPanelHeight + 15, 80, 20, "0", clrWhite, clrBlack); + CreateEditField("CP_CallStrike1", ControlPanelX + 280, ControlPanelY + topPanelHeight + 15, 80, 20, "0", clrWhite, clrBlack); + CreateEditField("CP_CallStrike2", ControlPanelX + 370, ControlPanelY + topPanelHeight + 15, 80, 20, "0", clrWhite, clrBlack); + CreateEditField("CP_CallStrike3", ControlPanelX + 460, ControlPanelY + topPanelHeight + 15, 80, 20, "0", clrWhite, clrBlack); + CreateEditField("CP_PutStrike1", ControlPanelX + 280, ControlPanelY + topPanelHeight + 45, 80, 20, "0", clrWhite, clrBlack); + CreateEditField("CP_PutStrike2", ControlPanelX + 370, ControlPanelY + topPanelHeight + 45, 80, 20, "0", clrWhite, clrBlack); + CreateEditField("CP_PutStrike3", ControlPanelX + 460, ControlPanelY + topPanelHeight + 45, 80, 20, "0", clrWhite, clrBlack); + CreateButton("CP_UpdateOI", ControlPanelX + 750, ControlPanelY + topPanelHeight + 25, 90, 35, "Update OI Data", clrCyan, clrBlack); +} + +void CreatePanel(string name, int x, int y, int width, int height, color bgColor, int borderType) { + if(ObjectFind(chart_id, name) < 0) { + ObjectCreate(chart_id, name, OBJ_RECTANGLE_LABEL, 0, 0, 0, 0, 0); + } + ObjectSetInteger(chart_id, name, OBJPROP_XDISTANCE, x); + ObjectSetInteger(chart_id, name, OBJPROP_YDISTANCE, y); + ObjectSetInteger(chart_id, name, OBJPROP_XSIZE, width); + ObjectSetInteger(chart_id, name, OBJPROP_YSIZE, height); + ObjectSetInteger(chart_id, name, OBJPROP_BGCOLOR, bgColor); + ObjectSetInteger(chart_id, name, OBJPROP_BORDER_TYPE, borderType); + ObjectSetInteger(chart_id, name, OBJPROP_ZORDER, 1000); +} + +void CreateButton(string name, int x, int y, int width, int height, string text, color bgColor, color textColor) { + if(ObjectFind(chart_id, name) < 0) { + ObjectCreate(chart_id, name, OBJ_BUTTON, 0, 0, 0, 0, 0); + } + ObjectSetInteger(chart_id, name, OBJPROP_XDISTANCE, x); + ObjectSetInteger(chart_id, name, OBJPROP_YDISTANCE, y); + ObjectSetInteger(chart_id, name, OBJPROP_XSIZE, width); + ObjectSetInteger(chart_id, name, OBJPROP_YSIZE, height); + ObjectSetString(chart_id, name, OBJPROP_TEXT, text); + ObjectSetInteger(chart_id, name, OBJPROP_BGCOLOR, bgColor); + ObjectSetInteger(chart_id, name, OBJPROP_COLOR, textColor); + ObjectSetInteger(chart_id, name, OBJPROP_FONTSIZE, 10); + ObjectSetInteger(chart_id, name, OBJPROP_ZORDER, 1001); +} + +void CreateLabel(string name, int x, int y, string text, color textColor, int fontSize) { + if(ObjectFind(chart_id, name) < 0) { + ObjectCreate(chart_id, name, OBJ_LABEL, 0, 0, 0, 0, 0); + } + ObjectSetInteger(chart_id, name, OBJPROP_XDISTANCE, x); + ObjectSetInteger(chart_id, name, OBJPROP_YDISTANCE, y); + ObjectSetString(chart_id, name, OBJPROP_TEXT, text); + ObjectSetInteger(chart_id, name, OBJPROP_COLOR, textColor); + ObjectSetInteger(chart_id, name, OBJPROP_FONTSIZE, fontSize); + ObjectSetInteger(chart_id, name, OBJPROP_ZORDER, 1002); +} + +void CreateEditField(string name, int x, int y, int width, int height, string defaultText, color bgColor, color textColor) { + if(ObjectFind(chart_id, name) < 0) { + ObjectCreate(chart_id, name, OBJ_EDIT, 0, 0, 0, 0, 0); + } + ObjectSetInteger(chart_id, name, OBJPROP_XDISTANCE, x); + ObjectSetInteger(chart_id, name, OBJPROP_YDISTANCE, y); + ObjectSetInteger(chart_id, name, OBJPROP_XSIZE, width); + ObjectSetInteger(chart_id, name, OBJPROP_YSIZE, height); + ObjectSetString(chart_id, name, OBJPROP_TEXT, defaultText); + ObjectSetInteger(chart_id, name, OBJPROP_BGCOLOR, bgColor); + ObjectSetInteger(chart_id, name, OBJPROP_COLOR, textColor); + ObjectSetInteger(chart_id, name, OBJPROP_FONTSIZE, 10); + ObjectSetInteger(chart_id, name, OBJPROP_ZORDER, 1003); +} + +void UpdateLabel(string name, int x, int y, string text, color textColor, int fontSize) { + if(ObjectFind(chart_id, name) < 0) { + CreateLabel(name, x, y, text, textColor, fontSize); + } else { + ObjectSetInteger(chart_id, name, OBJPROP_XDISTANCE, x); + ObjectSetInteger(chart_id, name, OBJPROP_YDISTANCE, y); + ObjectSetString(chart_id, name, OBJPROP_TEXT, text); + ObjectSetInteger(chart_id, name, OBJPROP_COLOR, textColor); + ObjectSetInteger(chart_id, name, OBJPROP_FONTSIZE, fontSize); + } +} + +void UpdateControlPanel() { + UpdateEditField("CP_FuturePrice", DoubleToString(DynamicFuturePrice, 2)); + UpdateEditField("CP_CallStrike1", DoubleToString(DynamicCallStrike1, 2)); + UpdateEditField("CP_CallStrike2", DoubleToString(DynamicCallStrike2, 2)); + UpdateEditField("CP_CallStrike3", DoubleToString(DynamicCallStrike3, 2)); + UpdateEditField("CP_PutStrike1", DoubleToString(DynamicPutStrike1, 2)); + UpdateEditField("CP_PutStrike2", DoubleToString(DynamicPutStrike2, 2)); + UpdateEditField("CP_PutStrike3", DoubleToString(DynamicPutStrike3, 2)); +} + +void UpdateEditField(string name, string value) { + if(ObjectFind(chart_id, name) >= 0) { + ObjectSetString(chart_id, name, OBJPROP_TEXT, value); + } +} + +void UpdateControlPanelValues() { + UpdateEditField("CP_FuturePrice", DoubleToString(DynamicFuturePrice, 2)); + UpdateEditField("CP_CallStrike1", DoubleToString(DynamicCallStrike1, 2)); + UpdateEditField("CP_CallStrike2", DoubleToString(DynamicCallStrike2, 2)); + UpdateEditField("CP_CallStrike3", DoubleToString(DynamicCallStrike3, 2)); + UpdateEditField("CP_PutStrike1", DoubleToString(DynamicPutStrike1, 2)); + UpdateEditField("CP_PutStrike2", DoubleToString(DynamicPutStrike2, 2)); + UpdateEditField("CP_PutStrike3", DoubleToString(DynamicPutStrike3, 2)); +} + +void HandleControlPanelClick(string name) { + if(name == "CP_CloseAll") { + CloseAllPositions(); + } + else if(name == "CP_UpdateOI") { + UpdateInputValues(); + InitializeOILevels(); + InitializeKeyLevels(); + Print("OI Levels Updated"); + } +} + +void UpdateInputValues() { + DynamicFuturePrice = StringToDouble(ObjectGetString(chart_id, "CP_FuturePrice", OBJPROP_TEXT)); + DynamicCallStrike1 = StringToDouble(ObjectGetString(chart_id, "CP_CallStrike1", OBJPROP_TEXT)); + DynamicCallStrike2 = StringToDouble(ObjectGetString(chart_id, "CP_CallStrike2", OBJPROP_TEXT)); + DynamicCallStrike3 = StringToDouble(ObjectGetString(chart_id, "CP_CallStrike3", OBJPROP_TEXT)); + DynamicPutStrike1 = StringToDouble(ObjectGetString(chart_id, "CP_PutStrike1", OBJPROP_TEXT)); + DynamicPutStrike2 = StringToDouble(ObjectGetString(chart_id, "CP_PutStrike2", OBJPROP_TEXT)); + DynamicPutStrike3 = StringToDouble(ObjectGetString(chart_id, "CP_PutStrike3", OBJPROP_TEXT)); +} diff --git a/oi_scraper/.env.example b/oi_scraper/.env.example index 5a47651..14658ed 100644 --- a/oi_scraper/.env.example +++ b/oi_scraper/.env.example @@ -1,15 +1,10 @@ # CME Group QuikStrike Login Credentials CME_USERNAME=your_username_here CME_PASSWORD=your_password_here +CME_LOGIN_URL=https://login.cmegroup.com/sso/accountstatus/showAuth.action -# Product Configuration -# Gold (XAUUSD/COMEX Gold - OG|GC): pid=40 -# Default product for XAUUSD trading -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool - -# Alternative products: -# SOFR (3M SOFR): https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool -# Silver: https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool +# QuikStrike URL (fixed - always same page) +QUIKSTRIKE_URL=https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html # Gold Price Source (investing.com) INVESTING_URL=https://www.investing.com/commodities/gold diff --git a/oi_scraper/README.md b/oi_scraper/README.md index 508f647..f9b1900 100644 --- a/oi_scraper/README.md +++ b/oi_scraper/README.md @@ -92,6 +92,7 @@ FuturePrice,4345.50 Edit `.env` to customize: - `PRODUCT_URL` - QuikStrike product page URL (requires login) +- `CME_LOGIN_URL` - CME login page URL (default: SSO URL) - `TOP_N_STRIKES` - Number of top strikes to export (default: 3) - `HEADLESS` - Run browser in headless mode (default: false for debugging) - `CSV_OUTPUT_PATH` - Output CSV file path diff --git a/oi_scraper/main.py b/oi_scraper/main.py index a5ba0e6..ee19ca0 100644 --- a/oi_scraper/main.py +++ b/oi_scraper/main.py @@ -10,9 +10,14 @@ load_dotenv() # Configuration CME_USERNAME = os.getenv("CME_USERNAME") CME_PASSWORD = os.getenv("CME_PASSWORD") -PRODUCT_URL = os.getenv( - "PRODUCT_URL", - "https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool", +CME_LOGIN_URL = os.getenv( + "CME_LOGIN_URL", "https://login.cmegroup.com/sso/accountstatus/showAuth.action" +) +QUIKSTRIKE_URL = ( + "https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html" +) +QUIKSTRIKE_URL = ( + "https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html" ) INVESTING_URL = os.getenv("INVESTING_URL", "https://www.investing.com/commodities/gold") CSV_OUTPUT_PATH = os.getenv("CSV_OUTPUT_PATH", "./oi_data.csv") @@ -44,28 +49,53 @@ def load_cookies(context): return False -def is_logged_in(page): - page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000) - page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) - return "login" not in page.url.lower() +def delete_cookies(): + if os.path.exists(COOKIE_FILE): + os.remove(COOKIE_FILE) + logger.info("Cookies deleted") + + +def are_cookies_valid(page): + logger.info("Checking if cookies are valid...") + page.goto(QUIKSTRIKE_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_timeout(3000) + + try: + frame = page.frame_locator("iframe.cmeIframe").first + page.wait_for_timeout(5000) + + table_exists = frame.locator("table.grid-thm").count() > 0 + if table_exists: + logger.info("Cookies are valid - OI table found in iframe") + else: + logger.info("Cookies may be expired - no OI table found in iframe") + return table_exists + except Exception as e: + logger.info(f"Cookies expired - error checking iframe: {e}") + return False def login_to_cme(page): logger.info("Attempting to login to CME QuikStrike...") - page.goto( - "https://www.cmegroup.com/account/login.html", timeout=TIMEOUT_SECONDS * 1000 - ) + page.goto(CME_LOGIN_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_timeout(1000) try: - page.fill('input[name="username"]', CME_USERNAME) - page.fill('input[name="password"]', CME_PASSWORD) - page.click('button[type="submit"]') + page.fill("#user", CME_USERNAME) + page.fill("#pwd", CME_PASSWORD) + page.click("#loginBtn") - page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + logger.info("Waiting for login redirect...") + page.wait_for_timeout(30000) - if "login" in page.url.lower(): - logger.error("Login failed - still on login page") + current_url = page.url.lower() + logger.info(f"Current URL after login attempt: {current_url}") + + if "login" in current_url or "sso" in current_url: + logger.error("Login may have failed - still on SSO/login page") page.screenshot(path="login_failed.png") return False @@ -79,20 +109,59 @@ def login_to_cme(page): return False +def select_gold_product(page): + logger.info("Selecting Gold product...") + + logger.info("Switching to iframe context...") + frame = page.frame_locator("iframe.cmeIframe").first + page.wait_for_timeout(5000) + + logger.info("Step 1: Clicking dropdown arrow...") + frame.locator("#ctl11_hlProductArrow").click() + page.wait_for_timeout(1000) + + logger.info("Step 2: Clicking Metals...") + frame.locator('a[groupid="6"]:has-text("Metals")').click() + page.wait_for_timeout(500) + + logger.info("Step 3: Clicking Precious Metals...") + frame.locator('a[familyid="6"]:has-text("Precious Metals")').click() + page.wait_for_timeout(500) + + logger.info("Step 4: Clicking Gold...") + frame.locator('a[title="Gold"]').click() + + logger.info("Waiting for Gold data to load...") + page.wait_for_timeout(10000) + logger.info("Gold product selected") + + def navigate_to_oi_heatmap(page): - logger.info(f"Navigating to OI Heatmap: {PRODUCT_URL}") - page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000) - page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + logger.info(f"Navigating to QuikStrike: {QUIKSTRIKE_URL}") + page.goto(QUIKSTRIKE_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_timeout(5000) + + select_gold_product(page) def extract_oi_data(page): logger.info("Extracting OI data from Gold matrix table...") + logger.info("Switching to iframe context...") + frame = page.frame_locator("iframe.cmeIframe").first + page.wait_for_timeout(8000) + + logger.info("Looking for table.grid-thm...") call_levels = [] put_levels = [] - table = page.locator("table.grid-thm").first + table = frame.locator("table.grid-thm").first + table.wait_for(state="visible", timeout=10000) + logger.info("Table found, waiting for data...") + rows = table.locator("tbody tr").all() + logger.info(f"Found {len(rows)} rows in table") for row in rows: try: @@ -100,31 +169,36 @@ def extract_oi_data(page): if len(cells) < 3: continue - strike_cell = cells[0].text_content().strip() - if not strike_cell or not strike_cell.replace(".", "").isdigit(): - continue - - strike = float(strike_cell) - - cells_with_data = cells[2:] - - for i in range(0, len(cells_with_data), 2): - if i + 1 >= len(cells_with_data): + strike = None + for cell in cells: + text = cell.text_content().strip() + if text and text.replace(".", "").isdigit(): + strike = float(text) break - call_cell = cells_with_data[i] - put_cell = cells_with_data[i + 1] + if strike is None: + continue + + number_cells = row.locator("td.number").all() + logger.debug(f"Strike {strike}: found {len(number_cells)} number cells") + + for i in range(0, len(number_cells), 2): + if i + 1 >= len(number_cells): + break + + call_cell = number_cells[i] + put_cell = number_cells[i + 1] call_text = call_cell.text_content().strip() put_text = put_cell.text_content().strip() - if call_text and call_text.replace(",", "").isdigit(): + if call_text and call_text != "-": call_oi = int(call_text.replace(",", "")) call_levels.append( {"Type": "CALL", "Strike": strike, "OI": call_oi} ) - if put_text and put_text.replace(",", "").isdigit(): + if put_text and put_text != "-": put_oi = int(put_text.replace(",", "")) put_levels.append({"Type": "PUT", "Strike": strike, "OI": put_oi}) @@ -132,25 +206,27 @@ def extract_oi_data(page): logger.warning(f"Error parsing row: {e}") continue - if not call_levels: - logger.warning("No CALL OI data extracted") - if not put_levels: - logger.warning("No PUT OI data extracted") + logger.info( + f"Extracted {len(call_levels)} CALL levels, {len(put_levels)} PUT levels" + ) - call_df = ( - pd.DataFrame(call_levels).nlargest(TOP_N_STRIKES, "OI") - if call_levels - else pd.DataFrame() - ) - put_df = ( - pd.DataFrame(put_levels).nlargest(TOP_N_STRIKES, "OI") - if put_levels - else pd.DataFrame() - ) + if call_levels: + call_df = pd.DataFrame(call_levels) + call_df = call_df.groupby("Strike", as_index=False).agg({"OI": "max"}) + call_df = call_df.nlargest(TOP_N_STRIKES, "OI") + else: + call_df = pd.DataFrame() + + if put_levels: + put_df = pd.DataFrame(put_levels) + put_df = put_df.groupby("Strike", as_index=False).agg({"OI": "max"}) + put_df = put_df.nlargest(TOP_N_STRIKES, "OI") + else: + put_df = pd.DataFrame() result_df = pd.concat([call_df, put_df], ignore_index=True) - logger.info(f"Extracted {len(result_df)} OI levels") + logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted") return result_df @@ -158,29 +234,39 @@ def scrape_investing_gold_price(page): logger.info(f"Scraping gold price from: {INVESTING_URL}") try: - page.goto(INVESTING_URL, timeout=TIMEOUT_SECONDS * 1000) - page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000) + page.goto(INVESTING_URL, timeout=60000, wait_until="domcontentloaded") + logger.info(f"Page loaded, title: {page.title()}") - price_locator = page.locator('div[data-test="instrument-price-last"]') + page.wait_for_timeout(5000) + logger.info("Waited for JavaScript to render") - if price_locator.count() > 0: - price_text = price_locator.text_content().strip() - price_text = price_text.replace(",", "") - price = float(price_text) - logger.info(f"Extracted gold price: {price}") - return price - else: - logger.warning("Price element not found, trying alternative selector") - alt_locator = page.locator(".text-5xl\\/9") - if alt_locator.count() > 0: - price_text = alt_locator.text_content().strip() - price_text = price_text.replace(",", "") - price = float(price_text) - logger.info(f"Extracted gold price (alt): {price}") - return price + selectors = [ + 'div[data-test="instrument-price-last"]', + ".text-5xl\\/9.font-bold.text-\\[#232526\\]", + '[data-test="instrument-price-last"]', + ".text-5xl\\/9", + ] - logger.warning("Could not extract gold price") - return 0.0 + price = 0.0 + for selector in selectors: + try: + locator = page.locator(selector) + if locator.count() > 0: + locator.first.wait_for(state="visible", timeout=10000) + price_text = locator.first.text_content().strip() + if price_text: + price_text = price_text.replace(",", "") + price = float(price_text) + logger.info(f"Extracted gold price ({selector}): {price}") + break + except Exception as e: + logger.debug(f"Selector {selector} failed: {e}") + continue + + if price == 0.0: + logger.warning("Could not extract gold price, all selectors failed") + + return price except Exception as e: logger.error(f"Error scraping gold price: {e}") @@ -212,13 +298,20 @@ def run_scraper(): context = browser.new_context() page = context.new_page() - loaded_cookies = load_cookies(context) - page2 = context.new_page() + cookies_loaded = load_cookies(context) + cookies_valid = False - if loaded_cookies and is_logged_in(page2): - logger.info("Using existing session (cookies)") + if cookies_loaded: + cookies_valid = are_cookies_valid(page) + + if cookies_valid: + logger.info("Using cached session") else: - logger.info("No valid session found, logging in...") + if cookies_loaded: + logger.info("Cookies expired, deleting and re-logging in...") + delete_cookies() + + logger.info("Logging in to CME...") if not login_to_cme(page): browser.close() if attempt < RETRY_ATTEMPTS - 1: @@ -229,14 +322,15 @@ def run_scraper(): else: logger.error("All login attempts failed") return - save_cookies(context) navigate_to_oi_heatmap(page) oi_data = extract_oi_data(page) + save_cookies(context) if not oi_data.empty: logger.info("Extracting gold price from investing.com...") future_price = scrape_investing_gold_price(page) + logger.info(f"Gold price extracted: {future_price}") export_to_csv(oi_data, future_price) else: