From b7c0e68fa8f4d4ac38ccb7a33369e217e48bd8e0 Mon Sep 17 00:00:00 2001 From: Kunthawat Greethong Date: Tue, 6 Jan 2026 20:18:12 +0700 Subject: [PATCH] refactor(oi): improve data extraction and consolidate documentation - Fix MQL5 API usage in EA to use correct CopyRates and POSITION_TYPE enums - Refactor scraper data extraction to use drop_duplicates for unique strikes - Consolidate Windows setup guide into main README - Add virtual environment batch files for easier setup and execution - Simplify run_scraper.bat to focus on core execution - Normalize lot calculation to use SymbolInfo.LotsStep() --- OI_OrderFlow_Absorption_XAUUSD.mq5 | 137 +++--- oi_scraper/README.md | 347 +++++++++------ oi_scraper/WINDOWS_SETUP.md | 658 ----------------------------- oi_scraper/main.py | 25 +- oi_scraper/run_scheduled.bat | 13 + oi_scraper/run_scraper.bat | 46 +- oi_scraper/run_with_venv.bat | 21 + oi_scraper/setup_env.bat | 34 ++ 8 files changed, 386 insertions(+), 895 deletions(-) delete mode 100644 oi_scraper/WINDOWS_SETUP.md create mode 100644 oi_scraper/run_scheduled.bat create mode 100644 oi_scraper/run_with_venv.bat create mode 100644 oi_scraper/setup_env.bat diff --git a/OI_OrderFlow_Absorption_XAUUSD.mq5 b/OI_OrderFlow_Absorption_XAUUSD.mq5 index b976b73..8f77632 100644 --- a/OI_OrderFlow_Absorption_XAUUSD.mq5 +++ b/OI_OrderFlow_Absorption_XAUUSD.mq5 @@ -358,64 +358,66 @@ void OnNewM1Bar(datetime newBarTime) { } void CalculateVolumeEMAFromHistory() { - double volumeArray[]; - ArraySetAsSeries(volumeArray, true); - CopyVolume(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, volumeArray); - - double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1); - double sum = 0; - - for(int i = 0; i < InpVolumeEmaPeriod; i++) { - sum += volumeArray[i]; - } - double avgVolume = sum / InpVolumeEmaPeriod; - - if(VolumeEmaValue == 0) { - VolumeEmaValue = avgVolume; - } else { - VolumeEmaValue = emaAlpha * avgVolume + (1 - emaAlpha) * VolumeEmaValue; - } + MqlRates rates[]; + ArraySetAsSeries(rates, true); + int copied = CopyRates(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, rates); + if(copied < InpVolumeEmaPeriod + 1) return; + + double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1); + double sum = 0; + + for(int i = 0; i < InpVolumeEmaPeriod; i++) { + sum += (double)rates[i].tick_volume; + } + double avgVolume = sum / InpVolumeEmaPeriod; + + if(VolumeEmaValue == 0) { + VolumeEmaValue = avgVolume; + } else { + VolumeEmaValue = emaAlpha * avgVolume + (1 - emaAlpha) * VolumeEmaValue; + } } void DetectAbsorptionFromHistory() { - double volumeArray[]; - ArraySetAsSeries(volumeArray, true); - CopyVolume(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, volumeArray); - - double avgVolume = 0; - for(int i = 0; i < InpAbsorptionBars; i++) { - avgVolume += volumeArray[i]; - } - avgVolume /= InpAbsorptionBars; - - double volumeThreshold = avgVolume * InpMinVolumeMultiplier; - - int sellAbsorptionCount = 0; - int buyAbsorptionCount = 0; - - for(int i = 1; i <= InpAbsorptionBars; i++) { - double high = iHigh(_Symbol, PERIOD_M1, i); - double low = iLow(_Symbol, PERIOD_M1, i); - double close = iClose(_Symbol, PERIOD_M1, i); - double open = iOpen(_Symbol, PERIOD_M1, i); - int barVolume = (int)iVolume(_Symbol, PERIOD_M1, i); - - double barRange = high - low; - double barDrift = close - open; - - bool highVolume = barVolume > volumeThreshold; - bool lowDrift = barRange < InpMaxPriceDriftPoints * _Point; - - if(highVolume && lowDrift && barDrift < 0) { - sellAbsorptionCount++; - } - - if(highVolume && lowDrift && barDrift > 0) { - buyAbsorptionCount++; - } - } - - int requiredBars = MathCeil(InpAbsorptionBars / 2.0); + MqlRates rates[]; + ArraySetAsSeries(rates, true); + int copied = CopyRates(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, rates); + if(copied < InpAbsorptionBars + 1) return; + + double avgVolume = 0; + for(int i = 0; i < InpAbsorptionBars; i++) { + avgVolume += (double)rates[i].tick_volume; + } + avgVolume /= InpAbsorptionBars; + + double volumeThreshold = avgVolume * InpMinVolumeMultiplier; + + int sellAbsorptionCount = 0; + int buyAbsorptionCount = 0; + + for(int i = 1; i <= InpAbsorptionBars; i++) { + double high = rates[i].high; + double low = rates[i].low; + double close = rates[i].close; + double open = rates[i].open; + int barVolume = (int)rates[i].tick_volume; + + double barRange = high - low; + double barDrift = close - open; + + bool highVolume = barVolume > volumeThreshold; + bool lowDrift = barRange < InpMaxPriceDriftPoints * _Point; + + if(highVolume && lowDrift && barDrift < 0) { + sellAbsorptionCount++; + } + + if(highVolume && lowDrift && barDrift > 0) { + buyAbsorptionCount++; + } + } + + int requiredBars = (int)MathCeil(InpAbsorptionBars / 2.0); if(sellAbsorptionCount >= requiredBars && IsPriceNearPutStrike()) { CurrentAbsorptionState = ABSORPTION_SELL; @@ -679,8 +681,8 @@ bool IsAtSupport() { } void ExecuteBuyTrade() { - double lotSize = CalculateLotSize(ORDER_TYPE_BUY); - if(lotSize <= 0) return; + double lotSize = CalculateLotSize(POSITION_TYPE_BUY); + if(lotSize <= 0) return; double sl = 0, tp = 0; double nearestPutStrike = GetNearestPutStrike(); @@ -715,8 +717,8 @@ void ExecuteBuyTrade() { } void ExecuteSellTrade() { - double lotSize = CalculateLotSize(ORDER_TYPE_SELL); - if(lotSize <= 0) return; + double lotSize = CalculateLotSize(POSITION_TYPE_SELL); + if(lotSize <= 0) return; double sl = 0, tp = 0; double nearestCallStrike = GetNearestCallStrike(); @@ -820,8 +822,8 @@ double CalculateLotSize(ENUM_POSITION_TYPE tradeType) { } double NormalizeLot(double lot) { - double step = SymbolInfo.LotStep(); - return MathFloor(lot / step) * step; + double step = SymbolInfo.LotsStep(); + return MathFloor(lot / step) * step; } void ManagePositions() { @@ -1037,8 +1039,11 @@ void CreateDashboard() { } void UpdateDashboard() { - UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8); - UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8); + MqlRates rates[]; + int copied = CopyRates(_Symbol, PERIOD_M1, 0, 1, rates); + + UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8); + UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8); string deltaText = DoubleToString(OrderFlowDeltaPercent, 1) + "%"; color deltaColor = OrderFlowDeltaPercent > 0 ? clrLime : (OrderFlowDeltaPercent < 0 ? clrRed : clrGray); @@ -1063,9 +1068,9 @@ void UpdateDashboard() { UpdateLabel("DB_Absorption", 20, 105, "Absorption: " + absorptionText, absorptionColor, 8); - int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (int)Volume(0); - UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) + - " (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8); + int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (copied > 0 ? (int)rates[0].tick_volume : 0); + UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) + + " (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8); string driftText = DoubleToString(PriceDrift / _Point, 1) + " pts"; color driftColor = PriceDrift < InpMaxPriceDriftPoints * _Point ? clrLime : clrOrange; diff --git a/oi_scraper/README.md b/oi_scraper/README.md index f9b1900..06a50bc 100644 --- a/oi_scraper/README.md +++ b/oi_scraper/README.md @@ -1,179 +1,268 @@ # CME OI Scraper -Python scraper to pull Open Interest data from CME Group QuikStrike and current gold price from investing.com. +Python scraper that extracts Open Interest data from CME Group QuikStrike and current gold price from investing.com. ## What It Extracts 1. **OI Levels (from CME QuikStrike):** - - Top 3 CALL strikes by OI volume - - Top 3 PUT strikes by OI volume + - Top 3 CALL strikes by OI volume (unique strikes) + - Top 3 PUT strikes by OI volume (unique strikes) 2. **Gold Price (from investing.com):** - - Current gold futures price (e.g., 4345.50) + - Current gold futures price (e.g., 4476.50) ## Prerequisites -- Python 3.9+ -- CME Group QuikStrike account with login credentials +- Python 3.9 or higher +- CME Group QuikStrike account (free registration at https://www.cmegroup.com) +- Windows 10/11 (for batch files) or Linux/macOS -## Installation +## Quick Start -1. Copy environment variables: -```bash -cp .env.example .env -``` +### Windows -2. Edit `.env` and add your CME credentials: -```bash -CME_USERNAME=your_username -CME_PASSWORD=your_password -``` +1. **Run one-time setup:** + ```cmd + cd C:\Path\To\oi_scraper + setup_env.bat + ``` -3. Install dependencies: -```bash -pip install -r requirements.txt -playwright install chromium -``` +2. **Run the scraper:** + ```cmd + run_with_venv.bat + ``` -## Usage +### Linux/macOS -### Basic Scraping +1. **Setup:** + ```bash + cd /path/to/oi_scraper + python3 -m venv venv + source venv/bin/activate + pip install -r requirements.txt + playwright install chromium + ``` -```bash -python main.py -``` - -This will: -- Login to CME QuikStrike -- Navigate to OI Heatmap -- Extract top 3 CALL and PUT strikes by OI volume -- Scrape current gold price from investing.com -- Export to `oi_data.csv` - -### Session Persistence - -The scraper automatically saves your login session to `cookies.json`. This means: - -- **First run**: Logs in with your credentials, saves cookies -- **Subsequent runs**: Uses saved cookies if session is still valid -- **Session expired**: Automatically logs in again and saves new cookies - -Benefits for scheduled runs: -- Faster execution (skips login when session is valid) -- Reduces login attempts to CME servers -- CME sessions typically last several days/weeks - -To force a fresh login, delete `cookies.json`: -```bash -rm cookies.json -``` - -### Output Format - -The CSV output is compatible with the EA's `LoadOIFromCSV()` and `LoadFuturePriceFromCSV()` functions: - -```csv -Type,Strike,OI -CALL,4345,155398 -CALL,4350,229137 -CALL,4360,90649 -PUT,4300,227936 -PUT,4290,270135 -PUT,4280,65839 - -[Price] -FuturePrice,4345.50 -``` - -**Note:** The `[Price]` section contains the current gold futures price scraped from investing.com. The EA reads this value for Delta calculation. +2. **Run:** + ```bash + source venv/bin/activate + python main.py + ``` ## Configuration -Edit `.env` to customize: +### Edit `.env` File -- `PRODUCT_URL` - QuikStrike product page URL (requires login) -- `CME_LOGIN_URL` - CME login page URL (default: SSO URL) -- `TOP_N_STRIKES` - Number of top strikes to export (default: 3) -- `HEADLESS` - Run browser in headless mode (default: false for debugging) -- `CSV_OUTPUT_PATH` - Output CSV file path -- `TIMEOUT_SECONDS` - Page load timeout +Copy and edit the environment file: -### Available Products - -**Gold (XAUUSD/COMEX Gold - OG|GC):** -``` -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool +```cmd +copy .env.example .env +notepad .env ``` -**Silver:** -``` -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool +Required settings: +```env +CME_USERNAME=your_cme_username +CME_PASSWORD=your_cme_password ``` -**SOFR (3M SOFR):** -``` -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +Optional settings: +```env +# Number of top strikes to export (default: 3) +TOP_N_STRIKES=3 + +# Run browser without window (default: false) +HEADLESS=false + +# Page timeout in seconds (default: 30) +TIMEOUT_SECONDS=30 + +# Output CSV path +CSV_OUTPUT_PATH=./oi_data.csv + +# Logging level: DEBUG, INFO, WARNING, ERROR +LOG_LEVEL=INFO ``` -**Note:** You must be logged in to access QuikStrike data. The scraper will automatically login using credentials from `.env`. +## Output Format + +The scraper exports to `oi_data.csv`: + +```csv +Type,Strike,OI +CALL,4375.0,147 +CALL,4450.0,173 +CALL,4500.0,176 +PUT,4435.0,49 +PUT,4400.0,102 +PUT,4515.0,150 + +[Price] +FuturePrice,4467.8 +``` + +The `[Price]` section contains the current gold futures price scraped from investing.com. + +## Session Persistence + +The scraper saves login sessions to `cookies.json`: + +- **First run:** Logs in with credentials, saves cookies +- **Subsequent runs:** Uses saved cookies if session is valid +- **Session expired:** Automatically re-logs in and saves new cookies + +This makes scheduled runs faster and reduces login attempts to CME servers. + +To force a fresh login: +```cmd +del cookies.json +``` ## Integration with EA -The EA reads OI data from CSV when `InpOISource = OI_SOURCE_CSV_FILE`. +The EA reads OI data from CSV when configured: +```mql5 +input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE; +``` -Place the generated `oi_data.csv` in MetaTrader's `MQL5/Files` directory. +Copy `oi_data.csv` to your MT5 `MQL5/Files` directory: +``` +C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\Common\MQL5\Files\oi_data.csv +``` -## Scheduling +## Automatic Daily Scheduling -Use cron or Windows Task Scheduler to run periodically: +### Windows Task Scheduler + +1. **Create scheduled task:** + - Open Task Scheduler (`taskschd.msc`) + - Click "Create Task" + +2. **Configure General tab:** + - Name: `CME OI Scraper - Daily` + - ✅ Run whether user is logged on or not + - ✅ Run with highest privileges + +3. **Configure Triggers tab:** + - New → On a schedule → Daily + - Start time: 9:00 AM (or your preferred time) + - ✅ Enabled + +4. **Configure Actions tab:** + - Action: Start a program + - Program/script: + ``` + C:\Path\To\oi_scraper\run_scheduled.bat + ``` + - Start in: + ``` + C:\Path\To\oi_scraper + ``` + +5. **Click OK to save** + +### Linux/macOS (cron) ```bash -# Run every hour -0 * * * * cd /path/to/oi_scraper && python main.py +# Edit crontab +crontab -e + +# Add line to run every day at 9 AM +0 9 * * * cd /path/to/oi_scraper && /path/to/venv/bin/python main.py ``` +## Batch Files Reference + +| File | Purpose | +|------|---------| +| `setup_env.bat` | One-time setup (creates virtual environment) | +| `run_with_venv.bat` | Manual run with visible window | +| `run_scheduled.bat` | For Task Scheduler (no window, no pause) | + ## Troubleshooting -**Login fails:** +### Module Not Found Errors + +**Error:** `ModuleNotFoundError: No module named 'playwright'` + +**Solution:** +```cmd +run_with_venv.bat +``` + +The virtual environment ensures all dependencies are isolated. + +### Login Fails + - Verify credentials in `.env` -- Check if CME requires 2FA -- Set `HEADLESS=false` to see what's happening -- Check screenshots: `login_failed.png`, `login_error.png`, `login_success.png` +- Check if CME requires 2FA (manual intervention needed) +- Set `HEADLESS=false` to see browser activity +- Check screenshots: `login_failed.png`, `login_error.png` -**No data extracted:** -- Check if table structure changed -- Increase `TIMEOUT_SECONDS` -- Check logs for detailed errors -- Screenshot saved as `login_debug.png` or `login_failed.png` +### No Data Extracted -**Login page selectors changed:** -- If the scraper can't find username/password inputs, CME may have updated their login page -- Update the selectors in `login_to_cme()` function in `main.py`: - ```python - # Example: update to match current CME login form - page.fill('input[id="username"]', CME_USERNAME) - page.fill('input[id="password"]', CME_PASSWORD) - page.click('button[type="submit"]') - ``` +- Check if CME table structure changed +- Increase `TIMEOUT_SECONDS=60` in `.env` +- Check logs for errors +- Screenshot saved as `login_debug.png` -**Browser issues:** -- Install Chromium dependencies: `playwright install chromium` -- Try different browser: Change `p.chromium.launch()` to `p.firefox.launch()` +### Browser Issues + +```cmd +# Reinstall Chromium +python -m playwright install chromium +``` + +### Session Expires Frequently + +Delete cookies to force fresh login: +```cmd +del cookies.json +``` + +### Check Python Path Issues (Windows) + +```cmd +# Check which Python is being used +where python + +# Use Python launcher +py -3 main.py + +# Or use the virtual environment +run_with_venv.bat +``` + +## Finding Product IDs + +To scrape other instruments (Silver, Crude Oil, etc.): + +1. Visit CME QuikStrike OI Heatmap +2. Login to your CME account +3. Select a product from the dropdown +4. The URL updates with the `pid` parameter +5. Note: This scraper is configured for Gold by default ## Notes -- The scraper targets the OI Heatmap table structure -- Only exports top N strikes by OI volume -- Login session is not persisted (login each run) -- Cookies could be saved for faster subsequent runs +- Targets the OI Heatmap table structure +- Exports top N unique strikes by OI volume +- Uses session cookies for faster subsequent runs +- CME sessions typically last several days to weeks +- Virtual environment recommended to avoid Python path conflicts -### Finding Product IDs +## Files -To find product IDs for other instruments: -1. Visit https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html -2. Login to your CME account -3. Select a product from the "Products" menu -4. The URL will update with the `pid` parameter -5. Copy that URL to your `.env` file - -Example: `https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html?pid=40` (Gold) \ No newline at end of file +``` +oi_scraper/ +├── main.py # Main scraper script +├── requirements.txt # Python dependencies +├── .env.example # Environment template +├── .env # Your credentials (create from example) +├── setup_env.bat # Windows: Create virtual environment +├── run_with_venv.bat # Windows: Manual run +├── run_scheduled.bat # Windows: Task Scheduler run +├── oi_data.csv # Output file (generated) +├── cookies.json # Session cookies (generated) +└── scraper.log # Log file (generated) +``` diff --git a/oi_scraper/WINDOWS_SETUP.md b/oi_scraper/WINDOWS_SETUP.md deleted file mode 100644 index 9e06a8a..0000000 --- a/oi_scraper/WINDOWS_SETUP.md +++ /dev/null @@ -1,658 +0,0 @@ -# CME OI Scraper - Windows Setup Guide - -Complete guide for setting up and running the CME OI scraper on Windows with automatic daily updates. - -## Table of Contents - -- [Prerequisites](#prerequisites) -- [Installation](#installation) -- [Configuration](#configuration) -- [Manual Testing](#manual-testing) -- [Automatic Daily Updates](#automatic-daily-updates) -- [MetaTrader 5 Integration](#metatrader-5-integration) -- [Troubleshooting](#troubleshooting) - ---- - -## Prerequisites - -### Required Software - -1. **Python 3.9 or higher** - - Download: https://www.python.org/downloads/ - - During installation: ✅ Check "Add Python to PATH" - -2. **CME Group QuikStrike Account** - - Free account required: https://www.cmegroup.com/ - - Register for QuikStrike access - - Save your username and password - -3. **MetaTrader 5** (for EA integration) - - Download: https://www.metatrader5.com/ - - Install on your Windows machine - -### Verify Python Installation - -```cmd -python --version -``` - -Expected output: `Python 3.9.x` or higher - -If not found, install Python or use `py` or `python3` commands. - ---- - -## Installation - -### Step 1: Navigate to Scraper Directory - -Open Command Prompt (cmd) and navigate: - -```cmd -cd C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper -``` - -Replace `YourUsername` with your actual Windows username. - -### Step 2: Create Environment File - -```cmd -copy .env.example .env -``` - -### Step 3: Edit .env File - -Open `.env` with Notepad: - -```cmd -notepad .env -``` - -Update with your credentials: - -```env -# CME Group QuikStrike Login Credentials -CME_USERNAME=your_actual_username_here -CME_PASSWORD=your_actual_password_here - -# Product Configuration (Gold) -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool - -# Output Settings -CSV_OUTPUT_PATH=./oi_data.csv -TOP_N_STRIKES=3 - -# Scraping Settings -HEADLESS=false -TIMEOUT_SECONDS=30 -RETRY_ATTEMPTS=3 - -# Logging -LOG_LEVEL=INFO -``` - -**Save and close** (Ctrl+S, then Alt+F4). - -### Step 4: Install Python Dependencies - -```cmd -pip install -r requirements.txt -``` - -Expected output: Successfully installed playwright, python-dotenv, pandas - -### Step 5: Install Playwright Browser - -```cmd -playwright install chromium -``` - -Expected output: Downloading Chromium... [progress bar] - ---- - -## Configuration - -### Available Products - -**Gold (XAUUSD/COMEX Gold - OG|GC):** -```env -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool -``` - -**Silver:** -```env -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool -``` - -**SOFR (3M SOFR):** -```env -PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool -``` - -### Configuration Options - -| Setting | Description | Default | -|----------|-------------|---------| -| `TOP_N_STRIKES` | Number of top strikes to export | 3 | -| `HEADLESS` | Run browser without window (true/false) | false | -| `TIMEOUT_SECONDS` | Page load timeout in seconds | 30 | -| `CSV_OUTPUT_PATH` | Output CSV file path | ./oi_data.csv | -| `LOG_LEVEL` | DEBUG, INFO, WARNING, ERROR | INFO | - ---- - -## Manual Testing - -### Run Scraper Manually - -```cmd -python main.py -``` - -Expected output: -``` -INFO:__main__:Cookies loaded from file -INFO:__main__:Using existing session (cookies) -INFO:__main__:Navigating to OI Heatmap: https://... -INFO:__main__:Extracting OI data from Gold matrix table... -INFO:__main__:Extracted 6 OI levels -INFO:__main__:Exported OI data to ./oi_data.csv -``` - -### Check Output - -**1. Verify CSV created:** - -```cmd -dir oi_data.csv -``` - -**2. View CSV content:** - -```cmd -notepad oi_data.csv -``` - -Expected format: -```csv -Type,Strike,OI -CALL,4400,6193 -CALL,4300,3826 -CALL,4350,1983 -PUT,4400,5559 -PUT,4300,2988 -PUT,4350,1214 -``` - -### Check Logs - -```cmd -type scraper.log -``` - -Or view in Notepad: - -```cmd -notepad scraper.log -``` - ---- - -## Automatic Daily Updates - -### Option 1: Windows Task Scheduler (Recommended) - -#### Step 1: Create Batch File Wrapper - -Create `run_scraper.bat` in the oi_scraper directory: - -```cmd -@echo off -cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper - -echo Starting CME OI Scraper at %date% %time% >> scraper.log -echo ---------------------------------------- >> scraper.log - -python main.py >> scraper.log 2>&1 - -if %ERRORLEVEL% EQU 0 ( - echo %date% %time%: Scraper completed successfully >> scraper.log -) else ( - echo %date% %time%: Scraper failed with error %ERRORLEVEL% >> scraper.log -) - -echo ---------------------------------------- >> scraper.log -``` - -Replace `YourUsername` with your actual username. - -#### Step 2: Open Task Scheduler - -Press `Win + R`, type `taskschd.msc`, press Enter - -Or: Start → Windows Administrative Tools → Task Scheduler - -#### Step 3: Create Task - -1. Click **"Create Basic Task"** on right sidebar -2. **Name:** `CME OI Scraper - Daily` -3. **Description:** `Update OI data from CME QuikStrike every day at 9 AM` -4. Click **Next** - -#### Step 4: Set Trigger - -1. **Trigger:** Select "Daily" -2. **Start date:** Today's date -3. **Start time:** 9:00:00 AM (or your preferred time) -4. Click **Next** - -#### Step 5: Set Action - -1. **Action:** Select "Start a program" -2. **Program/script:** - ``` - C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.bat - ``` -3. **Start in (optional):** - ``` - C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper - ``` -4. Click **Next** - -#### Step 6: Finish - -1. Review settings -2. Check "Open the Properties dialog for this task when I click Finish" -3. Click **Finish** - -#### Step 7: Configure Advanced Settings (Optional) - -In the Properties dialog: - -- **General tab:** - - ✅ Run whether user is logged on or not - - ✅ Do not store password (if using Windows authentication) - - ✅ Run with highest privileges - -- **Conditions tab:** - - ✅ Start the task only if the computer is on AC power - - ✅ Stop if the computer switches to battery power - - ✅ Wake the computer to run this task - -- **Settings tab:** - - ✅ Allow task to be run on demand - - ❌ Stop the task if it runs longer than: 30 minutes - - ✅ If the task fails, restart every: 5 minutes (up to 3 times) - -Click **OK** to save settings. - -#### Step 8: Test Task - -1. In Task Scheduler, find "CME OI Scraper - Daily" -2. Right-click → **Run** -3. Check `scraper.log` after a minute: - ```cmd - type scraper.log - ``` - ---- - -### Option 2: PowerShell Script (Advanced) - -#### Step 1: Create PowerShell Script - -Save as `run_scraper.ps1`: - -```powershell -# Script configuration -$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper" -$logFile = "$scriptPath\scraper.log" -$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" - -# Navigate to script directory -cd $scriptPath - -try { - # Run Python scraper - Write-Output "$timestamp: Starting CME OI Scraper" | Add-Content $logFile - & python main.py *>> $logFile 2>&1 - - # Check if CSV was created - if (Test-Path "oi_data.csv") { - $fileInfo = Get-Item "oi_data.csv" - Write-Output "$timestamp: Scraper completed successfully (CSV updated: $($fileInfo.LastWriteTime))" | Add-Content $logFile - } else { - Write-Output "$timestamp: WARNING - CSV file not created" | Add-Content $logFile - } -} catch { - $errorMsg = $_.Exception.Message - Write-Output "$timestamp: ERROR - $errorMsg" | Add-Content $logFile - exit 1 -} -``` - -#### Step 2: Update Task Scheduler to Use PowerShell - -Same steps as Option 1, but: - -- **Program/script:** `powershell.exe` -- **Add arguments:** - ``` - -ExecutionPolicy Bypass -File "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.ps1" - ``` - ---- - -## MetaTrader 5 Integration - -### Find MT5 Files Directory - -MT5 data directory location: - -``` -C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Terminal_ID]\MQL5\Files\ -``` - -**To find your Terminal_ID:** - -1. Open MT5 -2. Click **File** → **Open Data Folder** -3. Navigate to `Terminal\[Your_Terminal_ID]\MQL5\Files\` - -### Update Batch File to Copy to MT5 - -Edit `run_scraper.bat`: - -```cmd -@echo off -cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper - -echo Starting CME OI Scraper at %date% %time% >> scraper.log -echo ---------------------------------------- >> scraper.log - -python main.py >> scraper.log 2>&1 - -if %ERRORLEVEL% EQU 0 ( - if exist oi_data.csv ( - echo Copying OI data to MT5... >> scraper.log - copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" - - if %ERRORLEVEL% EQU 0 ( - echo %date% %time%: Scraper completed - OI data copied to MT5 >> scraper.log - ) else ( - echo %date% %time%: ERROR - Failed to copy to MT5 >> scraper.log - ) - ) else ( - echo %date% %time%: ERROR - oi_data.csv not found >> scraper.log - ) -) else ( - echo %date% %time%: ERROR - Scraper failed with error %ERRORLEVEL% >> scraper.log -) - -echo ---------------------------------------- >> scraper.log -``` - -Replace `[Your_Terminal_ID]` with your actual MT5 terminal ID. - -### Update EA Configuration - -In your EA (`OI_MeanReversion_Pro_XAUUSD_A.mq5`), set: - -```mql5 -input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE; // Load from CSV file -``` - -The EA will automatically read `oi_data.csv` from its Files directory. - ---- - -## Troubleshooting - -### Python Not Found - -**Error:** `'python' is not recognized as an internal or external command` - -**Solutions:** - -1. Use full path to Python: - ```cmd - C:\Users\YourUsername\AppData\Local\Programs\Python\Python312\python.exe main.py - ``` - -2. Use `py` launcher: - ```cmd - py main.py - ``` - -3. Reinstall Python with "Add to PATH" option - -### Module Import Errors - -**Error:** `ModuleNotFoundError: No module named 'playwright'` - -**Solution:** -```cmd -pip install -r requirements.txt -``` - -### Login Fails - -**Error:** `Login failed - still on login page` - -**Solutions:** - -1. Check credentials in `.env` file: - ```cmd - notepad .env - ``` - -2. Check login screenshots: - - `login_failed.png` - Shows login page - - `login_error.png` - Shows error during login - - `login_success.png` - Confirms successful login - -3. Manually test login at: https://www.cmegroup.com/ - -4. Check if 2FA is required (CME may require additional authentication) - -### No Data Extracted - -**Warning:** `No CALL OI data extracted` or `No PUT OI data extracted` - -**Solutions:** - -1. Check if you're logged in: - - Delete `cookies.json` to force fresh login - - Run scraper manually with `HEADLESS=false` in `.env` - -2. Check if page structure changed: - - View screenshots to see actual page content - - Check if Gold product URL is correct - -3. Increase timeout: - ```env - TIMEOUT_SECONDS=60 - ``` - -### Task Not Running - -**Issue:** Task Scheduler doesn't execute the task - -**Solutions:** - -1. Check task history: - - Task Scheduler → Right-click task → Properties → History tab - - Look for errors in the log - -2. Test manually: - - Right-click task → Run - - Check `scraper.log` for output - -3. Check account permissions: - - Ensure task is set to run with your Windows account - - Check "Run whether user is logged on or not" - -4. Check Windows Event Viewer: - - Event Viewer → Windows Logs → Application - - Look for Task Scheduler errors - -### Session Expiration - -**Issue:** Session expires after some time - -**Solution:** -The scraper will automatically re-login when cookies expire. No manual action needed. - -To force fresh login: -```cmd -del cookies.json -``` - -### Check Logs - -**View recent logs:** - -```cmd -type scraper.log | more -``` - -**View last 20 lines:** - -```cmd -powershell "Get-Content scraper.log -Tail 20" -``` - -**Search for errors:** - -```cmd -findstr /C:"ERROR" scraper.log -``` - -### Verify CSV Output - -**Check if CSV is valid:** - -```cmd -python -c "import pandas as pd; print(pd.read_csv('oi_data.csv'))" -``` - -**Check file size:** - -```cmd -dir oi_data.csv -``` - ---- - -## Advanced Options - -### Run Multiple Times Per Day - -**Edit Task Scheduler Trigger:** - -1. Open task properties → Triggers tab -2. Edit existing trigger → Click "New" to add additional -3. Set different times: - - 9:00 AM - - 12:00 PM - - 3:00 PM - - 6:00 PM - -### Run on Market Days Only - -**Create separate batch file:** - -```cmd -@echo off -cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper - -REM Check if today is weekday (1=Monday, 5=Friday) -for /f "skip=1 tokens=*" %%a in ('wmic path win32_localtime get dayofweek /value') do set DAY=%%a - -if %DAY% LSS 1 goto END -if %DAY% GTR 5 goto END - -REM Run scraper -python main.py >> scraper.log 2>&1 - -:END -``` - -### Email Notifications - -**Use PowerShell to send email on completion:** - -```powershell -# Add to run_scraper.ps1 at the end -$smtpServer = "smtp.gmail.com" -$smtpPort = 587 -$smtpUser = "your_email@gmail.com" -$smtpPass = "your_password" -$from = "CME OI Scraper " -$to = "your_email@gmail.com" -$subject = "CME OI Scraper - %date%" - -if ($errorOccurred) { - $body = "CME OI Scraper failed. Check logs for details." -} else { - $body = "CME OI Scraper completed successfully.`n`nUpdated files:`n- oi_data.csv" -} - -$message = New-Object System.Net.Mail.MailMessage $from, $to -$message.Subject = $subject -$message.Body = $body - -$smtp = New-Object System.Net.Mail.SmtpClient $smtpServer, $smtpPort -$smtp.EnableSsl = $true -$smtp.Credentials = New-Object System.Net.NetworkCredential $smtpUser, $smtpPass - -$smtp.Send($message) -``` - ---- - -## Summary - -**Quick Start Checklist:** - -- [ ] Python 3.9+ installed -- [ ] CME QuikStrike account created -- [ ] `.env` file configured with credentials -- [ ] Dependencies installed (`pip install -r requirements.txt`) -- [ ] Playwright browser installed (`playwright install chromium`) -- [ ] Manual test successful (`python main.py`) -- [ ] `oi_data.csv` created and valid -- [ ] Task Scheduler task created -- [ ] Task tested manually -- [ ] CSV copied to MT5 Files directory -- [ ] EA configured to use CSV file - -**Daily Workflow:** - -1. Task Scheduler runs at 9:00 AM -2. Batch file executes Python scraper -3. Scraper logs in with saved cookies (or fresh login) -4. OI data extracted and saved to `oi_data.csv` -5. CSV copied to MT5 Files directory -6. EA reads updated OI data -7. EA uses new OI levels for trading - ---- - -## Support - -For issues or questions: - -1. Check `scraper.log` for detailed error messages -2. Review screenshots (login_failed.png, login_error.png) -3. Verify `.env` configuration -4. Test manually without Task Scheduler -5. Check Windows Event Viewer for system errors - ---- - -**Last Updated:** January 4, 2026 -**Version:** 1.0 -**Platform:** Windows 10/11 diff --git a/oi_scraper/main.py b/oi_scraper/main.py index ee19ca0..8798592 100644 --- a/oi_scraper/main.py +++ b/oi_scraper/main.py @@ -1,3 +1,11 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CME OI Scraper - Extracts Open Interest data from CME QuikStrike and gold price from investing.com +Usage: python main.py +Requires: pip install -r requirements.txt +""" + import os import logging import json @@ -212,19 +220,24 @@ def extract_oi_data(page): if call_levels: call_df = pd.DataFrame(call_levels) - call_df = call_df.groupby("Strike", as_index=False).agg({"OI": "max"}) - call_df = call_df.nlargest(TOP_N_STRIKES, "OI") + call_df = call_df.drop_duplicates(subset="Strike", keep="first") + call_df = call_df.sort_values("OI") + call_df = call_df.tail(TOP_N_STRIKES) + call_df["Type"] = "CALL" else: call_df = pd.DataFrame() if put_levels: put_df = pd.DataFrame(put_levels) - put_df = put_df.groupby("Strike", as_index=False).agg({"OI": "max"}) - put_df = put_df.nlargest(TOP_N_STRIKES, "OI") + put_df = put_df.drop_duplicates(subset="Strike", keep="first") + put_df = put_df.sort_values("OI") + put_df = put_df.tail(TOP_N_STRIKES) + put_df["Type"] = "PUT" else: put_df = pd.DataFrame() - result_df = pd.concat([call_df, put_df], ignore_index=True) + result_df = pd.concat([call_df, put_df]) + result_df = result_df[["Type", "Strike", "OI"]] logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted") return result_df @@ -327,7 +340,7 @@ def run_scraper(): oi_data = extract_oi_data(page) save_cookies(context) - if not oi_data.empty: + if len(oi_data) > 0: logger.info("Extracting gold price from investing.com...") future_price = scrape_investing_gold_price(page) logger.info(f"Gold price extracted: {future_price}") diff --git a/oi_scraper/run_scheduled.bat b/oi_scraper/run_scheduled.bat new file mode 100644 index 0000000..c3a373e --- /dev/null +++ b/oi_scraper/run_scheduled.bat @@ -0,0 +1,13 @@ +@echo off +REM ========================================== +REM CME OI Scraper - Scheduled Task Version +REM For use with Windows Task Scheduler +REM ========================================== + +REM Navigate to script directory +cd /d %~dp0 + +REM Activate virtual environment and run scraper (no pause) +call venv\Scripts\activate.bat +python main.py +exit %ERRORLEVEL% diff --git a/oi_scraper/run_scraper.bat b/oi_scraper/run_scraper.bat index 7356d73..a484bcb 100644 --- a/oi_scraper/run_scraper.bat +++ b/oi_scraper/run_scraper.bat @@ -1,47 +1,21 @@ @echo off REM ========================================== -REM CME OI Scraper - Automatic Daily Runner +REM CME OI Scraper - Run with Virtual Environment REM ========================================== -REM Change to script directory -cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper +REM Navigate to script directory +cd /d %~dp0 echo ========================================== -echo CME OI Scraper - Daily Update +echo CME OI Scraper echo ========================================== -echo Started at: %date% %time% -echo ========================================== >> scraper.log + +REM Activate virtual environment +call venv\Scripts\activate.bat REM Run Python scraper -python main.py >> scraper.log 2>&1 +python main.py -REM Check if scraper succeeded -if %ERRORLEVEL% EQU 0 ( - echo [%date% %time%] Scraper completed successfully >> scraper.log - - REM Check if CSV file was created - if exist oi_data.csv ( - echo [%date% %time%] CSV file created successfully >> scraper.log - - REM Copy to MetaTrader 5 Files directory - REM Update this path to your actual MT5 directory - copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" - - if %ERRORLEVEL% EQU 0 ( - echo [%date% %time%] CSV copied to MT5 Files directory >> scraper.log - ) else ( - echo [%date% %time%] ERROR: Failed to copy CSV to MT5 directory >> scraper.log - ) - ) else ( - echo [%date% %time%] WARNING: oi_data.csv not found >> scraper.log - ) -) else ( - echo [%date% %time%] ERROR: Scraper failed with error code %ERRORLEVEL% >> scraper.log -) - -echo ========================================== -echo Completed at: %date% %time% -echo ========================================== - -REM Keep window open for 5 seconds to see any errors +REM Pause for 5 seconds if running manually (not scheduled) +if "%1"=="--scheduled" goto :eof timeout /t 5 diff --git a/oi_scraper/run_with_venv.bat b/oi_scraper/run_with_venv.bat new file mode 100644 index 0000000..8119539 --- /dev/null +++ b/oi_scraper/run_with_venv.bat @@ -0,0 +1,21 @@ +@echo off +REM ========================================== +REM CME OI Scraper - Manual Run with Virtual Environment +REM ========================================== + +REM Navigate to script directory +cd /d %~dp0 + +echo ========================================== +echo CME OI Scraper - Manual Run +echo ========================================== + +REM Activate virtual environment +call venv\Scripts\activate.bat + +REM Run Python scraper +python main.py + +echo. +echo Scraper completed. Check oi_data.csv for results. +timeout /t 5 diff --git a/oi_scraper/setup_env.bat b/oi_scraper/setup_env.bat new file mode 100644 index 0000000..6c25181 --- /dev/null +++ b/oi_scraper/setup_env.bat @@ -0,0 +1,34 @@ +@echo off +REM ========================================== +REM CME OI Scraper - Virtual Environment Setup +REM ========================================== + +echo ========================================== +echo Setting up Python Virtual Environment +echo ========================================== + +REM Navigate to script directory +cd /d %~dp0 + +REM Create virtual environment +echo Creating virtual environment... +py -3 -m venv venv + +REM Activate virtual environment and install dependencies +echo Installing dependencies... +call venv\Scripts\activate.bat +pip install --upgrade pip +pip install -r requirements.txt + +REM Install playwright browser +echo Installing Playwright browser... +python -m playwright install chromium + +echo ========================================== +echo Setup Complete! +echo ========================================== +echo. +echo To run the scraper, use: run_with_venv.bat +echo. + +pause