feat(oi): add open interest scraper module

add new oi_scraper directory for collecting open interest data
and update the main EA to integrate with the scraper functionality

oi_scraper/.env.example

@@ -0,0 +1,27 @@
+# CME Group QuikStrike Login Credentials
+CME_USERNAME=your_username_here
+CME_PASSWORD=your_password_here
+
+# Product Configuration
+# Gold (XAUUSD/COMEX Gold - OG|GC): pid=40
+# Default product for XAUUSD trading
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
+
+# Alternative products:
+# SOFR (3M SOFR): https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool
+# Silver: https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool
+
+# Gold Price Source (investing.com)
+INVESTING_URL=https://www.investing.com/commodities/gold
+
+# Output Settings
+CSV_OUTPUT_PATH=./oi_data.csv
+TOP_N_STRIKES=3
+
+# Scraping Settings
+HEADLESS=false  # Set to true for production
+TIMEOUT_SECONDS=30
+RETRY_ATTEMPTS=3
+
+# Logging
+LOG_LEVEL=INFO  # DEBUG, INFO, WARNING, ERROR

oi_scraper/.gitignore

@@ -0,0 +1,31 @@
+# Python cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# Environment variables
+.env
+
+# Output files
+*.csv
+*.png
+*.log
+
+# Session data
+cookies.json
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db

oi_scraper/README.md

@@ -0,0 +1,178 @@
+# CME OI Scraper
+
+Python scraper to pull Open Interest data from CME Group QuikStrike and current gold price from investing.com.
+
+## What It Extracts
+
+1. **OI Levels (from CME QuikStrike):**
+   - Top 3 CALL strikes by OI volume
+   - Top 3 PUT strikes by OI volume
+
+2. **Gold Price (from investing.com):**
+   - Current gold futures price (e.g., 4345.50)
+
+## Prerequisites
+
+- Python 3.9+
+- CME Group QuikStrike account with login credentials
+
+## Installation
+
+1. Copy environment variables:
+```bash
+cp .env.example .env
+```
+
+2. Edit `.env` and add your CME credentials:
+```bash
+CME_USERNAME=your_username
+CME_PASSWORD=your_password
+```
+
+3. Install dependencies:
+```bash
+pip install -r requirements.txt
+playwright install chromium
+```
+
+## Usage
+
+### Basic Scraping
+
+```bash
+python main.py
+```
+
+This will:
+- Login to CME QuikStrike
+- Navigate to OI Heatmap
+- Extract top 3 CALL and PUT strikes by OI volume
+- Scrape current gold price from investing.com
+- Export to `oi_data.csv`
+
+### Session Persistence
+
+The scraper automatically saves your login session to `cookies.json`. This means:
+
+- **First run**: Logs in with your credentials, saves cookies
+- **Subsequent runs**: Uses saved cookies if session is still valid
+- **Session expired**: Automatically logs in again and saves new cookies
+
+Benefits for scheduled runs:
+- Faster execution (skips login when session is valid)
+- Reduces login attempts to CME servers
+- CME sessions typically last several days/weeks
+
+To force a fresh login, delete `cookies.json`:
+```bash
+rm cookies.json
+```
+
+### Output Format
+
+The CSV output is compatible with the EA's `LoadOIFromCSV()` and `LoadFuturePriceFromCSV()` functions:
+
+```csv
+Type,Strike,OI
+CALL,4345,155398
+CALL,4350,229137
+CALL,4360,90649
+PUT,4300,227936
+PUT,4290,270135
+PUT,4280,65839
+
+[Price]
+FuturePrice,4345.50
+```
+
+**Note:** The `[Price]` section contains the current gold futures price scraped from investing.com. The EA reads this value for Delta calculation.
+
+## Configuration
+
+Edit `.env` to customize:
+
+- `PRODUCT_URL` - QuikStrike product page URL (requires login)
+- `TOP_N_STRIKES` - Number of top strikes to export (default: 3)
+- `HEADLESS` - Run browser in headless mode (default: false for debugging)
+- `CSV_OUTPUT_PATH` - Output CSV file path
+- `TIMEOUT_SECONDS` - Page load timeout
+
+### Available Products
+
+**Gold (XAUUSD/COMEX Gold - OG|GC):**
+```
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
+```
+
+**Silver:**
+```
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool
+```
+
+**SOFR (3M SOFR):**
+```
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool
+```
+
+**Note:** You must be logged in to access QuikStrike data. The scraper will automatically login using credentials from `.env`.
+
+## Integration with EA
+
+The EA reads OI data from CSV when `InpOISource = OI_SOURCE_CSV_FILE`.
+
+Place the generated `oi_data.csv` in MetaTrader's `MQL5/Files` directory.
+
+## Scheduling
+
+Use cron or Windows Task Scheduler to run periodically:
+
+```bash
+# Run every hour
+0 * * * * cd /path/to/oi_scraper && python main.py
+```
+
+## Troubleshooting
+
+**Login fails:**
+- Verify credentials in `.env`
+- Check if CME requires 2FA
+- Set `HEADLESS=false` to see what's happening
+- Check screenshots: `login_failed.png`, `login_error.png`, `login_success.png`
+
+**No data extracted:**
+- Check if table structure changed
+- Increase `TIMEOUT_SECONDS`
+- Check logs for detailed errors
+- Screenshot saved as `login_debug.png` or `login_failed.png`
+
+**Login page selectors changed:**
+- If the scraper can't find username/password inputs, CME may have updated their login page
+- Update the selectors in `login_to_cme()` function in `main.py`:
+  ```python
+  # Example: update to match current CME login form
+  page.fill('input[id="username"]', CME_USERNAME)
+  page.fill('input[id="password"]', CME_PASSWORD)
+  page.click('button[type="submit"]')
+  ```
+
+**Browser issues:**
+- Install Chromium dependencies: `playwright install chromium`
+- Try different browser: Change `p.chromium.launch()` to `p.firefox.launch()`
+
+## Notes
+
+- The scraper targets the OI Heatmap table structure
+- Only exports top N strikes by OI volume
+- Login session is not persisted (login each run)
+- Cookies could be saved for faster subsequent runs
+
+### Finding Product IDs
+
+To find product IDs for other instruments:
+1. Visit https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html
+2. Login to your CME account
+3. Select a product from the "Products" menu
+4. The URL will update with the `pid` parameter
+5. Copy that URL to your `.env` file
+
+Example: `https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html?pid=40` (Gold)

oi_scraper/WINDOWS_SETUP.md

@@ -0,0 +1,658 @@
+# CME OI Scraper - Windows Setup Guide
+
+Complete guide for setting up and running the CME OI scraper on Windows with automatic daily updates.
+
+## Table of Contents
+
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+- [Configuration](#configuration)
+- [Manual Testing](#manual-testing)
+- [Automatic Daily Updates](#automatic-daily-updates)
+- [MetaTrader 5 Integration](#metatrader-5-integration)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Prerequisites
+
+### Required Software
+
+1. **Python 3.9 or higher**
+   - Download: https://www.python.org/downloads/
+   - During installation: ✅ Check "Add Python to PATH"
+
+2. **CME Group QuikStrike Account**
+   - Free account required: https://www.cmegroup.com/
+   - Register for QuikStrike access
+   - Save your username and password
+
+3. **MetaTrader 5** (for EA integration)
+   - Download: https://www.metatrader5.com/
+   - Install on your Windows machine
+
+### Verify Python Installation
+
+```cmd
+python --version
+```
+
+Expected output: `Python 3.9.x` or higher
+
+If not found, install Python or use `py` or `python3` commands.
+
+---
+
+## Installation
+
+### Step 1: Navigate to Scraper Directory
+
+Open Command Prompt (cmd) and navigate:
+
+```cmd
+cd C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
+```
+
+Replace `YourUsername` with your actual Windows username.
+
+### Step 2: Create Environment File
+
+```cmd
+copy .env.example .env
+```
+
+### Step 3: Edit .env File
+
+Open `.env` with Notepad:
+
+```cmd
+notepad .env
+```
+
+Update with your credentials:
+
+```env
+# CME Group QuikStrike Login Credentials
+CME_USERNAME=your_actual_username_here
+CME_PASSWORD=your_actual_password_here
+
+# Product Configuration (Gold)
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
+
+# Output Settings
+CSV_OUTPUT_PATH=./oi_data.csv
+TOP_N_STRIKES=3
+
+# Scraping Settings
+HEADLESS=false
+TIMEOUT_SECONDS=30
+RETRY_ATTEMPTS=3
+
+# Logging
+LOG_LEVEL=INFO
+```
+
+**Save and close** (Ctrl+S, then Alt+F4).
+
+### Step 4: Install Python Dependencies
+
+```cmd
+pip install -r requirements.txt
+```
+
+Expected output: Successfully installed playwright, python-dotenv, pandas
+
+### Step 5: Install Playwright Browser
+
+```cmd
+playwright install chromium
+```
+
+Expected output: Downloading Chromium... [progress bar]
+
+---
+
+## Configuration
+
+### Available Products
+
+**Gold (XAUUSD/COMEX Gold - OG|GC):**
+```env
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
+```
+
+**Silver:**
+```env
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool
+```
+
+**SOFR (3M SOFR):**
+```env
+PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool
+```
+
+### Configuration Options
+
+| Setting | Description | Default |
+|----------|-------------|---------|
+| `TOP_N_STRIKES` | Number of top strikes to export | 3 |
+| `HEADLESS` | Run browser without window (true/false) | false |
+| `TIMEOUT_SECONDS` | Page load timeout in seconds | 30 |
+| `CSV_OUTPUT_PATH` | Output CSV file path | ./oi_data.csv |
+| `LOG_LEVEL` | DEBUG, INFO, WARNING, ERROR | INFO |
+
+---
+
+## Manual Testing
+
+### Run Scraper Manually
+
+```cmd
+python main.py
+```
+
+Expected output:
+```
+INFO:__main__:Cookies loaded from file
+INFO:__main__:Using existing session (cookies)
+INFO:__main__:Navigating to OI Heatmap: https://...
+INFO:__main__:Extracting OI data from Gold matrix table...
+INFO:__main__:Extracted 6 OI levels
+INFO:__main__:Exported OI data to ./oi_data.csv
+```
+
+### Check Output
+
+**1. Verify CSV created:**
+
+```cmd
+dir oi_data.csv
+```
+
+**2. View CSV content:**
+
+```cmd
+notepad oi_data.csv
+```
+
+Expected format:
+```csv
+Type,Strike,OI
+CALL,4400,6193
+CALL,4300,3826
+CALL,4350,1983
+PUT,4400,5559
+PUT,4300,2988
+PUT,4350,1214
+```
+
+### Check Logs
+
+```cmd
+type scraper.log
+```
+
+Or view in Notepad:
+
+```cmd
+notepad scraper.log
+```
+
+---
+
+## Automatic Daily Updates
+
+### Option 1: Windows Task Scheduler (Recommended)
+
+#### Step 1: Create Batch File Wrapper
+
+Create `run_scraper.bat` in the oi_scraper directory:
+
+```cmd
+@echo off
+cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
+
+echo Starting CME OI Scraper at %date% %time% >> scraper.log
+echo ---------------------------------------- >> scraper.log
+
+python main.py >> scraper.log 2>&1
+
+if %ERRORLEVEL% EQU 0 (
+    echo %date% %time%: Scraper completed successfully >> scraper.log
+) else (
+    echo %date% %time%: Scraper failed with error %ERRORLEVEL% >> scraper.log
+)
+
+echo ---------------------------------------- >> scraper.log
+```
+
+Replace `YourUsername` with your actual username.
+
+#### Step 2: Open Task Scheduler
+
+Press `Win + R`, type `taskschd.msc`, press Enter
+
+Or: Start → Windows Administrative Tools → Task Scheduler
+
+#### Step 3: Create Task
+
+1. Click **"Create Basic Task"** on right sidebar
+2. **Name:** `CME OI Scraper - Daily`
+3. **Description:** `Update OI data from CME QuikStrike every day at 9 AM`
+4. Click **Next**
+
+#### Step 4: Set Trigger
+
+1. **Trigger:** Select "Daily"
+2. **Start date:** Today's date
+3. **Start time:** 9:00:00 AM (or your preferred time)
+4. Click **Next**
+
+#### Step 5: Set Action
+
+1. **Action:** Select "Start a program"
+2. **Program/script:** 
+   ```
+   C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.bat
+   ```
+3. **Start in (optional):** 
+   ```
+   C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
+   ```
+4. Click **Next**
+
+#### Step 6: Finish
+
+1. Review settings
+2. Check "Open the Properties dialog for this task when I click Finish"
+3. Click **Finish**
+
+#### Step 7: Configure Advanced Settings (Optional)
+
+In the Properties dialog:
+
+- **General tab:**
+  - ✅ Run whether user is logged on or not
+  - ✅ Do not store password (if using Windows authentication)
+  - ✅ Run with highest privileges
+
+- **Conditions tab:**
+  - ✅ Start the task only if the computer is on AC power
+  - ✅ Stop if the computer switches to battery power
+  - ✅ Wake the computer to run this task
+
+- **Settings tab:**
+  - ✅ Allow task to be run on demand
+  - ❌ Stop the task if it runs longer than: 30 minutes
+  - ✅ If the task fails, restart every: 5 minutes (up to 3 times)
+
+Click **OK** to save settings.
+
+#### Step 8: Test Task
+
+1. In Task Scheduler, find "CME OI Scraper - Daily"
+2. Right-click → **Run**
+3. Check `scraper.log` after a minute:
+   ```cmd
+   type scraper.log
+   ```
+
+---
+
+### Option 2: PowerShell Script (Advanced)
+
+#### Step 1: Create PowerShell Script
+
+Save as `run_scraper.ps1`:
+
+```powershell
+# Script configuration
+$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper"
+$logFile = "$scriptPath\scraper.log"
+$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+
+# Navigate to script directory
+cd $scriptPath
+
+try {
+    # Run Python scraper
+    Write-Output "$timestamp: Starting CME OI Scraper" | Add-Content $logFile
+    & python main.py *>> $logFile 2>&1
+    
+    # Check if CSV was created
+    if (Test-Path "oi_data.csv") {
+        $fileInfo = Get-Item "oi_data.csv"
+        Write-Output "$timestamp: Scraper completed successfully (CSV updated: $($fileInfo.LastWriteTime))" | Add-Content $logFile
+    } else {
+        Write-Output "$timestamp: WARNING - CSV file not created" | Add-Content $logFile
+    }
+} catch {
+    $errorMsg = $_.Exception.Message
+    Write-Output "$timestamp: ERROR - $errorMsg" | Add-Content $logFile
+    exit 1
+}
+```
+
+#### Step 2: Update Task Scheduler to Use PowerShell
+
+Same steps as Option 1, but:
+
+- **Program/script:** `powershell.exe`
+- **Add arguments:** 
+  ```
+  -ExecutionPolicy Bypass -File "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.ps1"
+  ```
+
+---
+
+## MetaTrader 5 Integration
+
+### Find MT5 Files Directory
+
+MT5 data directory location:
+
+```
+C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Terminal_ID]\MQL5\Files\
+```
+
+**To find your Terminal_ID:**
+
+1. Open MT5
+2. Click **File** → **Open Data Folder**
+3. Navigate to `Terminal\[Your_Terminal_ID]\MQL5\Files\`
+
+### Update Batch File to Copy to MT5
+
+Edit `run_scraper.bat`:
+
+```cmd
+@echo off
+cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
+
+echo Starting CME OI Scraper at %date% %time% >> scraper.log
+echo ---------------------------------------- >> scraper.log
+
+python main.py >> scraper.log 2>&1
+
+if %ERRORLEVEL% EQU 0 (
+    if exist oi_data.csv (
+        echo Copying OI data to MT5... >> scraper.log
+        copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
+        
+        if %ERRORLEVEL% EQU 0 (
+            echo %date% %time%: Scraper completed - OI data copied to MT5 >> scraper.log
+        ) else (
+            echo %date% %time%: ERROR - Failed to copy to MT5 >> scraper.log
+        )
+    ) else (
+        echo %date% %time%: ERROR - oi_data.csv not found >> scraper.log
+    )
+) else (
+    echo %date% %time%: ERROR - Scraper failed with error %ERRORLEVEL% >> scraper.log
+)
+
+echo ---------------------------------------- >> scraper.log
+```
+
+Replace `[Your_Terminal_ID]` with your actual MT5 terminal ID.
+
+### Update EA Configuration
+
+In your EA (`OI_MeanReversion_Pro_XAUUSD_A.mq5`), set:
+
+```mql5
+input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE;  // Load from CSV file
+```
+
+The EA will automatically read `oi_data.csv` from its Files directory.
+
+---
+
+## Troubleshooting
+
+### Python Not Found
+
+**Error:** `'python' is not recognized as an internal or external command`
+
+**Solutions:**
+
+1. Use full path to Python:
+   ```cmd
+   C:\Users\YourUsername\AppData\Local\Programs\Python\Python312\python.exe main.py
+   ```
+
+2. Use `py` launcher:
+   ```cmd
+   py main.py
+   ```
+
+3. Reinstall Python with "Add to PATH" option
+
+### Module Import Errors
+
+**Error:** `ModuleNotFoundError: No module named 'playwright'`
+
+**Solution:**
+```cmd
+pip install -r requirements.txt
+```
+
+### Login Fails
+
+**Error:** `Login failed - still on login page`
+
+**Solutions:**
+
+1. Check credentials in `.env` file:
+   ```cmd
+   notepad .env
+   ```
+
+2. Check login screenshots:
+   - `login_failed.png` - Shows login page
+   - `login_error.png` - Shows error during login
+   - `login_success.png` - Confirms successful login
+
+3. Manually test login at: https://www.cmegroup.com/
+
+4. Check if 2FA is required (CME may require additional authentication)
+
+### No Data Extracted
+
+**Warning:** `No CALL OI data extracted` or `No PUT OI data extracted`
+
+**Solutions:**
+
+1. Check if you're logged in:
+   - Delete `cookies.json` to force fresh login
+   - Run scraper manually with `HEADLESS=false` in `.env`
+
+2. Check if page structure changed:
+   - View screenshots to see actual page content
+   - Check if Gold product URL is correct
+
+3. Increase timeout:
+   ```env
+   TIMEOUT_SECONDS=60
+   ```
+
+### Task Not Running
+
+**Issue:** Task Scheduler doesn't execute the task
+
+**Solutions:**
+
+1. Check task history:
+   - Task Scheduler → Right-click task → Properties → History tab
+   - Look for errors in the log
+
+2. Test manually:
+   - Right-click task → Run
+   - Check `scraper.log` for output
+
+3. Check account permissions:
+   - Ensure task is set to run with your Windows account
+   - Check "Run whether user is logged on or not"
+
+4. Check Windows Event Viewer:
+   - Event Viewer → Windows Logs → Application
+   - Look for Task Scheduler errors
+
+### Session Expiration
+
+**Issue:** Session expires after some time
+
+**Solution:**
+The scraper will automatically re-login when cookies expire. No manual action needed.
+
+To force fresh login:
+```cmd
+del cookies.json
+```
+
+### Check Logs
+
+**View recent logs:**
+
+```cmd
+type scraper.log | more
+```
+
+**View last 20 lines:**
+
+```cmd
+powershell "Get-Content scraper.log -Tail 20"
+```
+
+**Search for errors:**
+
+```cmd
+findstr /C:"ERROR" scraper.log
+```
+
+### Verify CSV Output
+
+**Check if CSV is valid:**
+
+```cmd
+python -c "import pandas as pd; print(pd.read_csv('oi_data.csv'))"
+```
+
+**Check file size:**
+
+```cmd
+dir oi_data.csv
+```
+
+---
+
+## Advanced Options
+
+### Run Multiple Times Per Day
+
+**Edit Task Scheduler Trigger:**
+
+1. Open task properties → Triggers tab
+2. Edit existing trigger → Click "New" to add additional
+3. Set different times:
+   - 9:00 AM
+   - 12:00 PM
+   - 3:00 PM
+   - 6:00 PM
+
+### Run on Market Days Only
+
+**Create separate batch file:**
+
+```cmd
+@echo off
+cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
+
+REM Check if today is weekday (1=Monday, 5=Friday)
+for /f "skip=1 tokens=*" %%a in ('wmic path win32_localtime get dayofweek /value') do set DAY=%%a
+
+if %DAY% LSS 1 goto END
+if %DAY% GTR 5 goto END
+
+REM Run scraper
+python main.py >> scraper.log 2>&1
+
+:END
+```
+
+### Email Notifications
+
+**Use PowerShell to send email on completion:**
+
+```powershell
+# Add to run_scraper.ps1 at the end
+$smtpServer = "smtp.gmail.com"
+$smtpPort = 587
+$smtpUser = "your_email@gmail.com"
+$smtpPass = "your_password"
+$from = "CME OI Scraper <your_email@gmail.com>"
+$to = "your_email@gmail.com"
+$subject = "CME OI Scraper - %date%"
+
+if ($errorOccurred) {
+    $body = "CME OI Scraper failed. Check logs for details."
+} else {
+    $body = "CME OI Scraper completed successfully.`n`nUpdated files:`n- oi_data.csv"
+}
+
+$message = New-Object System.Net.Mail.MailMessage $from, $to
+$message.Subject = $subject
+$message.Body = $body
+
+$smtp = New-Object System.Net.Mail.SmtpClient $smtpServer, $smtpPort
+$smtp.EnableSsl = $true
+$smtp.Credentials = New-Object System.Net.NetworkCredential $smtpUser, $smtpPass
+
+$smtp.Send($message)
+```
+
+---
+
+## Summary
+
+**Quick Start Checklist:**
+
+- [ ] Python 3.9+ installed
+- [ ] CME QuikStrike account created
+- [ ] `.env` file configured with credentials
+- [ ] Dependencies installed (`pip install -r requirements.txt`)
+- [ ] Playwright browser installed (`playwright install chromium`)
+- [ ] Manual test successful (`python main.py`)
+- [ ] `oi_data.csv` created and valid
+- [ ] Task Scheduler task created
+- [ ] Task tested manually
+- [ ] CSV copied to MT5 Files directory
+- [ ] EA configured to use CSV file
+
+**Daily Workflow:**
+
+1. Task Scheduler runs at 9:00 AM
+2. Batch file executes Python scraper
+3. Scraper logs in with saved cookies (or fresh login)
+4. OI data extracted and saved to `oi_data.csv`
+5. CSV copied to MT5 Files directory
+6. EA reads updated OI data
+7. EA uses new OI levels for trading
+
+---
+
+## Support
+
+For issues or questions:
+
+1. Check `scraper.log` for detailed error messages
+2. Review screenshots (login_failed.png, login_error.png)
+3. Verify `.env` configuration
+4. Test manually without Task Scheduler
+5. Check Windows Event Viewer for system errors
+
+---
+
+**Last Updated:** January 4, 2026
+**Version:** 1.0
+**Platform:** Windows 10/11

oi_scraper/main.py

@@ -0,0 +1,257 @@
+import os
+import logging
+import json
+from playwright.sync_api import sync_playwright
+from dotenv import load_dotenv
+import pandas as pd
+
+load_dotenv()
+
+# Configuration
+CME_USERNAME = os.getenv("CME_USERNAME")
+CME_PASSWORD = os.getenv("CME_PASSWORD")
+PRODUCT_URL = os.getenv(
+    "PRODUCT_URL",
+    "https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool",
+)
+INVESTING_URL = os.getenv("INVESTING_URL", "https://www.investing.com/commodities/gold")
+CSV_OUTPUT_PATH = os.getenv("CSV_OUTPUT_PATH", "./oi_data.csv")
+TOP_N_STRIKES = int(os.getenv("TOP_N_STRIKES", "3"))
+HEADLESS = os.getenv("HEADLESS", "false").lower() == "true"
+TIMEOUT_SECONDS = int(os.getenv("TIMEOUT_SECONDS", "30"))
+RETRY_ATTEMPTS = int(os.getenv("RETRY_ATTEMPTS", "3"))
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+COOKIE_FILE = "./cookies.json"
+
+logging.basicConfig(level=getattr(logging, LOG_LEVEL))
+logger = logging.getLogger(__name__)
+
+
+def save_cookies(context):
+    cookies = context.cookies()
+    with open(COOKIE_FILE, "w") as f:
+        json.dump(cookies, f)
+    logger.info("Cookies saved to file")
+
+
+def load_cookies(context):
+    if os.path.exists(COOKIE_FILE):
+        with open(COOKIE_FILE, "r") as f:
+            cookies = json.load(f)
+        context.add_cookies(cookies)
+        logger.info("Cookies loaded from file")
+        return True
+    return False
+
+
+def is_logged_in(page):
+    page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000)
+    page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000)
+    return "login" not in page.url.lower()
+
+
+def login_to_cme(page):
+    logger.info("Attempting to login to CME QuikStrike...")
+
+    page.goto(
+        "https://www.cmegroup.com/account/login.html", timeout=TIMEOUT_SECONDS * 1000
+    )
+
+    try:
+        page.fill('input[name="username"]', CME_USERNAME)
+        page.fill('input[name="password"]', CME_PASSWORD)
+        page.click('button[type="submit"]')
+
+        page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000)
+
+        if "login" in page.url.lower():
+            logger.error("Login failed - still on login page")
+            page.screenshot(path="login_failed.png")
+            return False
+
+        logger.info("Login successful")
+        page.screenshot(path="login_success.png")
+        return True
+
+    except Exception as e:
+        logger.error(f"Login error: {e}")
+        page.screenshot(path="login_error.png")
+        return False
+
+
+def navigate_to_oi_heatmap(page):
+    logger.info(f"Navigating to OI Heatmap: {PRODUCT_URL}")
+    page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000)
+    page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000)
+
+
+def extract_oi_data(page):
+    logger.info("Extracting OI data from Gold matrix table...")
+
+    call_levels = []
+    put_levels = []
+
+    table = page.locator("table.grid-thm").first
+    rows = table.locator("tbody tr").all()
+
+    for row in rows:
+        try:
+            cells = row.locator("td").all()
+            if len(cells) < 3:
+                continue
+
+            strike_cell = cells[0].text_content().strip()
+            if not strike_cell or not strike_cell.replace(".", "").isdigit():
+                continue
+
+            strike = float(strike_cell)
+
+            cells_with_data = cells[2:]
+
+            for i in range(0, len(cells_with_data), 2):
+                if i + 1 >= len(cells_with_data):
+                    break
+
+                call_cell = cells_with_data[i]
+                put_cell = cells_with_data[i + 1]
+
+                call_text = call_cell.text_content().strip()
+                put_text = put_cell.text_content().strip()
+
+                if call_text and call_text.replace(",", "").isdigit():
+                    call_oi = int(call_text.replace(",", ""))
+                    call_levels.append(
+                        {"Type": "CALL", "Strike": strike, "OI": call_oi}
+                    )
+
+                if put_text and put_text.replace(",", "").isdigit():
+                    put_oi = int(put_text.replace(",", ""))
+                    put_levels.append({"Type": "PUT", "Strike": strike, "OI": put_oi})
+
+        except Exception as e:
+            logger.warning(f"Error parsing row: {e}")
+            continue
+
+    if not call_levels:
+        logger.warning("No CALL OI data extracted")
+    if not put_levels:
+        logger.warning("No PUT OI data extracted")
+
+    call_df = (
+        pd.DataFrame(call_levels).nlargest(TOP_N_STRIKES, "OI")
+        if call_levels
+        else pd.DataFrame()
+    )
+    put_df = (
+        pd.DataFrame(put_levels).nlargest(TOP_N_STRIKES, "OI")
+        if put_levels
+        else pd.DataFrame()
+    )
+
+    result_df = pd.concat([call_df, put_df], ignore_index=True)
+
+    logger.info(f"Extracted {len(result_df)} OI levels")
+    return result_df
+
+
+def scrape_investing_gold_price(page):
+    logger.info(f"Scraping gold price from: {INVESTING_URL}")
+
+    try:
+        page.goto(INVESTING_URL, timeout=TIMEOUT_SECONDS * 1000)
+        page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000)
+
+        price_locator = page.locator('div[data-test="instrument-price-last"]')
+
+        if price_locator.count() > 0:
+            price_text = price_locator.text_content().strip()
+            price_text = price_text.replace(",", "")
+            price = float(price_text)
+            logger.info(f"Extracted gold price: {price}")
+            return price
+        else:
+            logger.warning("Price element not found, trying alternative selector")
+            alt_locator = page.locator(".text-5xl\\/9")
+            if alt_locator.count() > 0:
+                price_text = alt_locator.text_content().strip()
+                price_text = price_text.replace(",", "")
+                price = float(price_text)
+                logger.info(f"Extracted gold price (alt): {price}")
+                return price
+
+        logger.warning("Could not extract gold price")
+        return 0.0
+
+    except Exception as e:
+        logger.error(f"Error scraping gold price: {e}")
+        return 0.0
+
+
+def export_to_csv(df, future_price=0.0):
+    output_path = CSV_OUTPUT_PATH
+
+    with open(output_path, "w") as f:
+        df.to_csv(f, index=False)
+        f.write("\n[Price]\n")
+        f.write(f"FuturePrice,{future_price}\n")
+
+    logger.info(f"Exported OI data and price to {output_path}")
+
+
+def run_scraper():
+    if not CME_USERNAME or not CME_PASSWORD:
+        logger.error("Missing CME_USERNAME or CME_PASSWORD in .env file")
+        return
+
+    future_price = 0.0
+
+    for attempt in range(RETRY_ATTEMPTS):
+        try:
+            with sync_playwright() as p:
+                browser = p.chromium.launch(headless=HEADLESS)
+                context = browser.new_context()
+                page = context.new_page()
+
+                loaded_cookies = load_cookies(context)
+                page2 = context.new_page()
+
+                if loaded_cookies and is_logged_in(page2):
+                    logger.info("Using existing session (cookies)")
+                else:
+                    logger.info("No valid session found, logging in...")
+                    if not login_to_cme(page):
+                        browser.close()
+                        if attempt < RETRY_ATTEMPTS - 1:
+                            logger.info(
+                                f"Retrying... Attempt {attempt + 2}/{RETRY_ATTEMPTS}"
+                            )
+                            continue
+                        else:
+                            logger.error("All login attempts failed")
+                            return
+                    save_cookies(context)
+
+                navigate_to_oi_heatmap(page)
+                oi_data = extract_oi_data(page)
+
+                if not oi_data.empty:
+                    logger.info("Extracting gold price from investing.com...")
+                    future_price = scrape_investing_gold_price(page)
+
+                    export_to_csv(oi_data, future_price)
+                else:
+                    logger.warning("No OI data extracted")
+
+                browser.close()
+                break
+
+        except Exception as e:
+            logger.error(f"Scraper error (attempt {attempt + 1}): {e}")
+            if attempt < RETRY_ATTEMPTS - 1:
+                logger.info(f"Retrying... Attempt {attempt + 2}/{RETRY_ATTEMPTS}")
+            else:
+                logger.error("All attempts failed")
+
+
+if __name__ == "__main__":
+    run_scraper()

oi_scraper/requirements.txt

@@ -0,0 +1,3 @@
+playwright>=1.40.0
+python-dotenv>=1.0.0
+pandas>=2.2.0

oi_scraper/run_scraper.bat

@@ -0,0 +1,47 @@
+@echo off
+REM ==========================================
+REM CME OI Scraper - Automatic Daily Runner
+REM ==========================================
+
+REM Change to script directory
+cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
+
+echo ==========================================
+echo CME OI Scraper - Daily Update
+echo ==========================================
+echo Started at: %date% %time%
+echo ========================================== >> scraper.log
+
+REM Run Python scraper
+python main.py >> scraper.log 2>&1
+
+REM Check if scraper succeeded
+if %ERRORLEVEL% EQU 0 (
+    echo [%date% %time%] Scraper completed successfully >> scraper.log
+    
+    REM Check if CSV file was created
+    if exist oi_data.csv (
+        echo [%date% %time%] CSV file created successfully >> scraper.log
+        
+        REM Copy to MetaTrader 5 Files directory
+        REM Update this path to your actual MT5 directory
+        copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
+        
+        if %ERRORLEVEL% EQU 0 (
+            echo [%date% %time%] CSV copied to MT5 Files directory >> scraper.log
+        ) else (
+            echo [%date% %time%] ERROR: Failed to copy CSV to MT5 directory >> scraper.log
+        )
+    ) else (
+        echo [%date% %time%] WARNING: oi_data.csv not found >> scraper.log
+    )
+) else (
+    echo [%date% %time%] ERROR: Scraper failed with error code %ERRORLEVEL% >> scraper.log
+)
+
+echo ==========================================
+echo Completed at: %date% %time%
+echo ==========================================
+
+REM Keep window open for 5 seconds to see any errors
+timeout /t 5

oi_scraper/run_scraper.ps1

@@ -0,0 +1,77 @@
+# CME OI Scraper - PowerShell Script
+# Copy this file to: run_scraper.ps1
+
+# ==========================================
+# Configuration
+# ==========================================
+$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper"
+$logFile = "$scriptPath\scraper.log"
+$csvFile = "$scriptPath\oi_data.csv"
+$mt5Path = "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
+
+# ==========================================
+# Helper Functions
+# ==========================================
+function Write-Log {
+    param([string]$message)
+    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+    $logEntry = "[$timestamp] $message"
+    Write-Output $logEntry | Add-Content $logFile
+    Write-Host $logEntry
+}
+
+# ==========================================
+# Main Script
+# ==========================================
+
+# Navigate to script directory
+cd $scriptPath
+
+Write-Log "=========================================="
+Write-Log "CME OI Scraper - Daily Update"
+Write-Log "=========================================="
+
+try {
+    # Run Python scraper
+    Write-Log "Starting Python scraper..."
+    & python main.py *>> $logFile 2>&1
+    $exitCode = $LASTEXITCODE
+    
+    if ($exitCode -eq 0) {
+        Write-Log "Python scraper completed successfully"
+        
+        # Check if CSV was created
+        if (Test-Path $csvFile)) {
+            $fileInfo = Get-Item $csvFile
+            Write-Log "CSV file found (Last modified: $($fileInfo.LastWriteTime))"
+            
+            # Copy to MT5 directory
+            Write-Log "Copying CSV to MetaTrader 5 Files directory..."
+            
+            try {
+                Copy-Item -Path $csvFile -Destination $mt5Path -Force
+                Write-Log "CSV successfully copied to MT5 directory"
+                
+                # Verify copy
+                if (Test-Path $mt5Path)) {
+                    Write-Log "Verified: MT5 CSV file exists"
+                } else {
+                    Write-Log "ERROR: MT5 CSV file not found after copy"
+                }
+            } catch {
+                Write-Log "ERROR: Failed to copy to MT5 directory - $_"
+            }
+        } else {
+            Write-Log "WARNING: CSV file not found after scraper execution"
+        }
+    } else {
+        Write-Log "ERROR: Python scraper failed with exit code $exitCode"
+    }
+} catch {
+    Write-Log "ERROR: Script failed - $($_.Exception.Message)"
+    exit 1
+}
+
+Write-Log "=========================================="
+Write-Log "Script completed"
+Write-Log "=========================================="