diff --git a/OI_MeanReversion_Pro_XAUUSD_A.mq5 b/OI_MeanReversion_Pro_XAUUSD_A.mq5 index 979b95f..66a7a9b 100644 Binary files a/OI_MeanReversion_Pro_XAUUSD_A.mq5 and b/OI_MeanReversion_Pro_XAUUSD_A.mq5 differ diff --git a/oi_scraper/.env.example b/oi_scraper/.env.example new file mode 100644 index 0000000..5a47651 --- /dev/null +++ b/oi_scraper/.env.example @@ -0,0 +1,27 @@ +# CME Group QuikStrike Login Credentials +CME_USERNAME=your_username_here +CME_PASSWORD=your_password_here + +# Product Configuration +# Gold (XAUUSD/COMEX Gold - OG|GC): pid=40 +# Default product for XAUUSD trading +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool + +# Alternative products: +# SOFR (3M SOFR): https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +# Silver: https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool + +# Gold Price Source (investing.com) +INVESTING_URL=https://www.investing.com/commodities/gold + +# Output Settings +CSV_OUTPUT_PATH=./oi_data.csv +TOP_N_STRIKES=3 + +# Scraping Settings +HEADLESS=false # Set to true for production +TIMEOUT_SECONDS=30 +RETRY_ATTEMPTS=3 + +# Logging +LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR \ No newline at end of file diff --git a/oi_scraper/.gitignore b/oi_scraper/.gitignore new file mode 100644 index 0000000..4b3fd0c --- /dev/null +++ b/oi_scraper/.gitignore @@ -0,0 +1,31 @@ +# Python cache +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +venv/ +env/ +ENV/ + +# Environment variables +.env + +# Output files +*.csv +*.png +*.log + +# Session data +cookies.json + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/oi_scraper/README.md b/oi_scraper/README.md new file mode 100644 index 0000000..508f647 --- /dev/null +++ b/oi_scraper/README.md @@ -0,0 +1,178 @@ +# CME OI Scraper + +Python scraper to pull Open Interest data from CME Group QuikStrike and current gold price from investing.com. + +## What It Extracts + +1. **OI Levels (from CME QuikStrike):** + - Top 3 CALL strikes by OI volume + - Top 3 PUT strikes by OI volume + +2. **Gold Price (from investing.com):** + - Current gold futures price (e.g., 4345.50) + +## Prerequisites + +- Python 3.9+ +- CME Group QuikStrike account with login credentials + +## Installation + +1. Copy environment variables: +```bash +cp .env.example .env +``` + +2. Edit `.env` and add your CME credentials: +```bash +CME_USERNAME=your_username +CME_PASSWORD=your_password +``` + +3. Install dependencies: +```bash +pip install -r requirements.txt +playwright install chromium +``` + +## Usage + +### Basic Scraping + +```bash +python main.py +``` + +This will: +- Login to CME QuikStrike +- Navigate to OI Heatmap +- Extract top 3 CALL and PUT strikes by OI volume +- Scrape current gold price from investing.com +- Export to `oi_data.csv` + +### Session Persistence + +The scraper automatically saves your login session to `cookies.json`. This means: + +- **First run**: Logs in with your credentials, saves cookies +- **Subsequent runs**: Uses saved cookies if session is still valid +- **Session expired**: Automatically logs in again and saves new cookies + +Benefits for scheduled runs: +- Faster execution (skips login when session is valid) +- Reduces login attempts to CME servers +- CME sessions typically last several days/weeks + +To force a fresh login, delete `cookies.json`: +```bash +rm cookies.json +``` + +### Output Format + +The CSV output is compatible with the EA's `LoadOIFromCSV()` and `LoadFuturePriceFromCSV()` functions: + +```csv +Type,Strike,OI +CALL,4345,155398 +CALL,4350,229137 +CALL,4360,90649 +PUT,4300,227936 +PUT,4290,270135 +PUT,4280,65839 + +[Price] +FuturePrice,4345.50 +``` + +**Note:** The `[Price]` section contains the current gold futures price scraped from investing.com. The EA reads this value for Delta calculation. + +## Configuration + +Edit `.env` to customize: + +- `PRODUCT_URL` - QuikStrike product page URL (requires login) +- `TOP_N_STRIKES` - Number of top strikes to export (default: 3) +- `HEADLESS` - Run browser in headless mode (default: false for debugging) +- `CSV_OUTPUT_PATH` - Output CSV file path +- `TIMEOUT_SECONDS` - Page load timeout + +### Available Products + +**Gold (XAUUSD/COMEX Gold - OG|GC):** +``` +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool +``` + +**Silver:** +``` +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool +``` + +**SOFR (3M SOFR):** +``` +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +``` + +**Note:** You must be logged in to access QuikStrike data. The scraper will automatically login using credentials from `.env`. + +## Integration with EA + +The EA reads OI data from CSV when `InpOISource = OI_SOURCE_CSV_FILE`. + +Place the generated `oi_data.csv` in MetaTrader's `MQL5/Files` directory. + +## Scheduling + +Use cron or Windows Task Scheduler to run periodically: + +```bash +# Run every hour +0 * * * * cd /path/to/oi_scraper && python main.py +``` + +## Troubleshooting + +**Login fails:** +- Verify credentials in `.env` +- Check if CME requires 2FA +- Set `HEADLESS=false` to see what's happening +- Check screenshots: `login_failed.png`, `login_error.png`, `login_success.png` + +**No data extracted:** +- Check if table structure changed +- Increase `TIMEOUT_SECONDS` +- Check logs for detailed errors +- Screenshot saved as `login_debug.png` or `login_failed.png` + +**Login page selectors changed:** +- If the scraper can't find username/password inputs, CME may have updated their login page +- Update the selectors in `login_to_cme()` function in `main.py`: + ```python + # Example: update to match current CME login form + page.fill('input[id="username"]', CME_USERNAME) + page.fill('input[id="password"]', CME_PASSWORD) + page.click('button[type="submit"]') + ``` + +**Browser issues:** +- Install Chromium dependencies: `playwright install chromium` +- Try different browser: Change `p.chromium.launch()` to `p.firefox.launch()` + +## Notes + +- The scraper targets the OI Heatmap table structure +- Only exports top N strikes by OI volume +- Login session is not persisted (login each run) +- Cookies could be saved for faster subsequent runs + +### Finding Product IDs + +To find product IDs for other instruments: +1. Visit https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html +2. Login to your CME account +3. Select a product from the "Products" menu +4. The URL will update with the `pid` parameter +5. Copy that URL to your `.env` file + +Example: `https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html?pid=40` (Gold) \ No newline at end of file diff --git a/oi_scraper/WINDOWS_SETUP.md b/oi_scraper/WINDOWS_SETUP.md new file mode 100644 index 0000000..9e06a8a --- /dev/null +++ b/oi_scraper/WINDOWS_SETUP.md @@ -0,0 +1,658 @@ +# CME OI Scraper - Windows Setup Guide + +Complete guide for setting up and running the CME OI scraper on Windows with automatic daily updates. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Installation](#installation) +- [Configuration](#configuration) +- [Manual Testing](#manual-testing) +- [Automatic Daily Updates](#automatic-daily-updates) +- [MetaTrader 5 Integration](#metatrader-5-integration) +- [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites + +### Required Software + +1. **Python 3.9 or higher** + - Download: https://www.python.org/downloads/ + - During installation: ✅ Check "Add Python to PATH" + +2. **CME Group QuikStrike Account** + - Free account required: https://www.cmegroup.com/ + - Register for QuikStrike access + - Save your username and password + +3. **MetaTrader 5** (for EA integration) + - Download: https://www.metatrader5.com/ + - Install on your Windows machine + +### Verify Python Installation + +```cmd +python --version +``` + +Expected output: `Python 3.9.x` or higher + +If not found, install Python or use `py` or `python3` commands. + +--- + +## Installation + +### Step 1: Navigate to Scraper Directory + +Open Command Prompt (cmd) and navigate: + +```cmd +cd C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper +``` + +Replace `YourUsername` with your actual Windows username. + +### Step 2: Create Environment File + +```cmd +copy .env.example .env +``` + +### Step 3: Edit .env File + +Open `.env` with Notepad: + +```cmd +notepad .env +``` + +Update with your credentials: + +```env +# CME Group QuikStrike Login Credentials +CME_USERNAME=your_actual_username_here +CME_PASSWORD=your_actual_password_here + +# Product Configuration (Gold) +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool + +# Output Settings +CSV_OUTPUT_PATH=./oi_data.csv +TOP_N_STRIKES=3 + +# Scraping Settings +HEADLESS=false +TIMEOUT_SECONDS=30 +RETRY_ATTEMPTS=3 + +# Logging +LOG_LEVEL=INFO +``` + +**Save and close** (Ctrl+S, then Alt+F4). + +### Step 4: Install Python Dependencies + +```cmd +pip install -r requirements.txt +``` + +Expected output: Successfully installed playwright, python-dotenv, pandas + +### Step 5: Install Playwright Browser + +```cmd +playwright install chromium +``` + +Expected output: Downloading Chromium... [progress bar] + +--- + +## Configuration + +### Available Products + +**Gold (XAUUSD/COMEX Gold - OG|GC):** +```env +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool +``` + +**Silver:** +```env +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool +``` + +**SOFR (3M SOFR):** +```env +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +``` + +### Configuration Options + +| Setting | Description | Default | +|----------|-------------|---------| +| `TOP_N_STRIKES` | Number of top strikes to export | 3 | +| `HEADLESS` | Run browser without window (true/false) | false | +| `TIMEOUT_SECONDS` | Page load timeout in seconds | 30 | +| `CSV_OUTPUT_PATH` | Output CSV file path | ./oi_data.csv | +| `LOG_LEVEL` | DEBUG, INFO, WARNING, ERROR | INFO | + +--- + +## Manual Testing + +### Run Scraper Manually + +```cmd +python main.py +``` + +Expected output: +``` +INFO:__main__:Cookies loaded from file +INFO:__main__:Using existing session (cookies) +INFO:__main__:Navigating to OI Heatmap: https://... +INFO:__main__:Extracting OI data from Gold matrix table... +INFO:__main__:Extracted 6 OI levels +INFO:__main__:Exported OI data to ./oi_data.csv +``` + +### Check Output + +**1. Verify CSV created:** + +```cmd +dir oi_data.csv +``` + +**2. View CSV content:** + +```cmd +notepad oi_data.csv +``` + +Expected format: +```csv +Type,Strike,OI +CALL,4400,6193 +CALL,4300,3826 +CALL,4350,1983 +PUT,4400,5559 +PUT,4300,2988 +PUT,4350,1214 +``` + +### Check Logs + +```cmd +type scraper.log +``` + +Or view in Notepad: + +```cmd +notepad scraper.log +``` + +--- + +## Automatic Daily Updates + +### Option 1: Windows Task Scheduler (Recommended) + +#### Step 1: Create Batch File Wrapper + +Create `run_scraper.bat` in the oi_scraper directory: + +```cmd +@echo off +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +echo Starting CME OI Scraper at %date% %time% >> scraper.log +echo ---------------------------------------- >> scraper.log + +python main.py >> scraper.log 2>&1 + +if %ERRORLEVEL% EQU 0 ( + echo %date% %time%: Scraper completed successfully >> scraper.log +) else ( + echo %date% %time%: Scraper failed with error %ERRORLEVEL% >> scraper.log +) + +echo ---------------------------------------- >> scraper.log +``` + +Replace `YourUsername` with your actual username. + +#### Step 2: Open Task Scheduler + +Press `Win + R`, type `taskschd.msc`, press Enter + +Or: Start → Windows Administrative Tools → Task Scheduler + +#### Step 3: Create Task + +1. Click **"Create Basic Task"** on right sidebar +2. **Name:** `CME OI Scraper - Daily` +3. **Description:** `Update OI data from CME QuikStrike every day at 9 AM` +4. Click **Next** + +#### Step 4: Set Trigger + +1. **Trigger:** Select "Daily" +2. **Start date:** Today's date +3. **Start time:** 9:00:00 AM (or your preferred time) +4. Click **Next** + +#### Step 5: Set Action + +1. **Action:** Select "Start a program" +2. **Program/script:** + ``` + C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.bat + ``` +3. **Start in (optional):** + ``` + C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + ``` +4. Click **Next** + +#### Step 6: Finish + +1. Review settings +2. Check "Open the Properties dialog for this task when I click Finish" +3. Click **Finish** + +#### Step 7: Configure Advanced Settings (Optional) + +In the Properties dialog: + +- **General tab:** + - ✅ Run whether user is logged on or not + - ✅ Do not store password (if using Windows authentication) + - ✅ Run with highest privileges + +- **Conditions tab:** + - ✅ Start the task only if the computer is on AC power + - ✅ Stop if the computer switches to battery power + - ✅ Wake the computer to run this task + +- **Settings tab:** + - ✅ Allow task to be run on demand + - ❌ Stop the task if it runs longer than: 30 minutes + - ✅ If the task fails, restart every: 5 minutes (up to 3 times) + +Click **OK** to save settings. + +#### Step 8: Test Task + +1. In Task Scheduler, find "CME OI Scraper - Daily" +2. Right-click → **Run** +3. Check `scraper.log` after a minute: + ```cmd + type scraper.log + ``` + +--- + +### Option 2: PowerShell Script (Advanced) + +#### Step 1: Create PowerShell Script + +Save as `run_scraper.ps1`: + +```powershell +# Script configuration +$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper" +$logFile = "$scriptPath\scraper.log" +$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + +# Navigate to script directory +cd $scriptPath + +try { + # Run Python scraper + Write-Output "$timestamp: Starting CME OI Scraper" | Add-Content $logFile + & python main.py *>> $logFile 2>&1 + + # Check if CSV was created + if (Test-Path "oi_data.csv") { + $fileInfo = Get-Item "oi_data.csv" + Write-Output "$timestamp: Scraper completed successfully (CSV updated: $($fileInfo.LastWriteTime))" | Add-Content $logFile + } else { + Write-Output "$timestamp: WARNING - CSV file not created" | Add-Content $logFile + } +} catch { + $errorMsg = $_.Exception.Message + Write-Output "$timestamp: ERROR - $errorMsg" | Add-Content $logFile + exit 1 +} +``` + +#### Step 2: Update Task Scheduler to Use PowerShell + +Same steps as Option 1, but: + +- **Program/script:** `powershell.exe` +- **Add arguments:** + ``` + -ExecutionPolicy Bypass -File "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.ps1" + ``` + +--- + +## MetaTrader 5 Integration + +### Find MT5 Files Directory + +MT5 data directory location: + +``` +C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Terminal_ID]\MQL5\Files\ +``` + +**To find your Terminal_ID:** + +1. Open MT5 +2. Click **File** → **Open Data Folder** +3. Navigate to `Terminal\[Your_Terminal_ID]\MQL5\Files\` + +### Update Batch File to Copy to MT5 + +Edit `run_scraper.bat`: + +```cmd +@echo off +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +echo Starting CME OI Scraper at %date% %time% >> scraper.log +echo ---------------------------------------- >> scraper.log + +python main.py >> scraper.log 2>&1 + +if %ERRORLEVEL% EQU 0 ( + if exist oi_data.csv ( + echo Copying OI data to MT5... >> scraper.log + copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" + + if %ERRORLEVEL% EQU 0 ( + echo %date% %time%: Scraper completed - OI data copied to MT5 >> scraper.log + ) else ( + echo %date% %time%: ERROR - Failed to copy to MT5 >> scraper.log + ) + ) else ( + echo %date% %time%: ERROR - oi_data.csv not found >> scraper.log + ) +) else ( + echo %date% %time%: ERROR - Scraper failed with error %ERRORLEVEL% >> scraper.log +) + +echo ---------------------------------------- >> scraper.log +``` + +Replace `[Your_Terminal_ID]` with your actual MT5 terminal ID. + +### Update EA Configuration + +In your EA (`OI_MeanReversion_Pro_XAUUSD_A.mq5`), set: + +```mql5 +input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE; // Load from CSV file +``` + +The EA will automatically read `oi_data.csv` from its Files directory. + +--- + +## Troubleshooting + +### Python Not Found + +**Error:** `'python' is not recognized as an internal or external command` + +**Solutions:** + +1. Use full path to Python: + ```cmd + C:\Users\YourUsername\AppData\Local\Programs\Python\Python312\python.exe main.py + ``` + +2. Use `py` launcher: + ```cmd + py main.py + ``` + +3. Reinstall Python with "Add to PATH" option + +### Module Import Errors + +**Error:** `ModuleNotFoundError: No module named 'playwright'` + +**Solution:** +```cmd +pip install -r requirements.txt +``` + +### Login Fails + +**Error:** `Login failed - still on login page` + +**Solutions:** + +1. Check credentials in `.env` file: + ```cmd + notepad .env + ``` + +2. Check login screenshots: + - `login_failed.png` - Shows login page + - `login_error.png` - Shows error during login + - `login_success.png` - Confirms successful login + +3. Manually test login at: https://www.cmegroup.com/ + +4. Check if 2FA is required (CME may require additional authentication) + +### No Data Extracted + +**Warning:** `No CALL OI data extracted` or `No PUT OI data extracted` + +**Solutions:** + +1. Check if you're logged in: + - Delete `cookies.json` to force fresh login + - Run scraper manually with `HEADLESS=false` in `.env` + +2. Check if page structure changed: + - View screenshots to see actual page content + - Check if Gold product URL is correct + +3. Increase timeout: + ```env + TIMEOUT_SECONDS=60 + ``` + +### Task Not Running + +**Issue:** Task Scheduler doesn't execute the task + +**Solutions:** + +1. Check task history: + - Task Scheduler → Right-click task → Properties → History tab + - Look for errors in the log + +2. Test manually: + - Right-click task → Run + - Check `scraper.log` for output + +3. Check account permissions: + - Ensure task is set to run with your Windows account + - Check "Run whether user is logged on or not" + +4. Check Windows Event Viewer: + - Event Viewer → Windows Logs → Application + - Look for Task Scheduler errors + +### Session Expiration + +**Issue:** Session expires after some time + +**Solution:** +The scraper will automatically re-login when cookies expire. No manual action needed. + +To force fresh login: +```cmd +del cookies.json +``` + +### Check Logs + +**View recent logs:** + +```cmd +type scraper.log | more +``` + +**View last 20 lines:** + +```cmd +powershell "Get-Content scraper.log -Tail 20" +``` + +**Search for errors:** + +```cmd +findstr /C:"ERROR" scraper.log +``` + +### Verify CSV Output + +**Check if CSV is valid:** + +```cmd +python -c "import pandas as pd; print(pd.read_csv('oi_data.csv'))" +``` + +**Check file size:** + +```cmd +dir oi_data.csv +``` + +--- + +## Advanced Options + +### Run Multiple Times Per Day + +**Edit Task Scheduler Trigger:** + +1. Open task properties → Triggers tab +2. Edit existing trigger → Click "New" to add additional +3. Set different times: + - 9:00 AM + - 12:00 PM + - 3:00 PM + - 6:00 PM + +### Run on Market Days Only + +**Create separate batch file:** + +```cmd +@echo off +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +REM Check if today is weekday (1=Monday, 5=Friday) +for /f "skip=1 tokens=*" %%a in ('wmic path win32_localtime get dayofweek /value') do set DAY=%%a + +if %DAY% LSS 1 goto END +if %DAY% GTR 5 goto END + +REM Run scraper +python main.py >> scraper.log 2>&1 + +:END +``` + +### Email Notifications + +**Use PowerShell to send email on completion:** + +```powershell +# Add to run_scraper.ps1 at the end +$smtpServer = "smtp.gmail.com" +$smtpPort = 587 +$smtpUser = "your_email@gmail.com" +$smtpPass = "your_password" +$from = "CME OI Scraper " +$to = "your_email@gmail.com" +$subject = "CME OI Scraper - %date%" + +if ($errorOccurred) { + $body = "CME OI Scraper failed. Check logs for details." +} else { + $body = "CME OI Scraper completed successfully.`n`nUpdated files:`n- oi_data.csv" +} + +$message = New-Object System.Net.Mail.MailMessage $from, $to +$message.Subject = $subject +$message.Body = $body + +$smtp = New-Object System.Net.Mail.SmtpClient $smtpServer, $smtpPort +$smtp.EnableSsl = $true +$smtp.Credentials = New-Object System.Net.NetworkCredential $smtpUser, $smtpPass + +$smtp.Send($message) +``` + +--- + +## Summary + +**Quick Start Checklist:** + +- [ ] Python 3.9+ installed +- [ ] CME QuikStrike account created +- [ ] `.env` file configured with credentials +- [ ] Dependencies installed (`pip install -r requirements.txt`) +- [ ] Playwright browser installed (`playwright install chromium`) +- [ ] Manual test successful (`python main.py`) +- [ ] `oi_data.csv` created and valid +- [ ] Task Scheduler task created +- [ ] Task tested manually +- [ ] CSV copied to MT5 Files directory +- [ ] EA configured to use CSV file + +**Daily Workflow:** + +1. Task Scheduler runs at 9:00 AM +2. Batch file executes Python scraper +3. Scraper logs in with saved cookies (or fresh login) +4. OI data extracted and saved to `oi_data.csv` +5. CSV copied to MT5 Files directory +6. EA reads updated OI data +7. EA uses new OI levels for trading + +--- + +## Support + +For issues or questions: + +1. Check `scraper.log` for detailed error messages +2. Review screenshots (login_failed.png, login_error.png) +3. Verify `.env` configuration +4. Test manually without Task Scheduler +5. Check Windows Event Viewer for system errors + +--- + +**Last Updated:** January 4, 2026 +**Version:** 1.0 +**Platform:** Windows 10/11 diff --git a/oi_scraper/main.py b/oi_scraper/main.py new file mode 100644 index 0000000..a5ba0e6 --- /dev/null +++ b/oi_scraper/main.py @@ -0,0 +1,257 @@ +import os +import logging +import json +from playwright.sync_api import sync_playwright +from dotenv import load_dotenv +import pandas as pd + +load_dotenv() + +# Configuration +CME_USERNAME = os.getenv("CME_USERNAME") +CME_PASSWORD = os.getenv("CME_PASSWORD") +PRODUCT_URL = os.getenv( + "PRODUCT_URL", + "https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool", +) +INVESTING_URL = os.getenv("INVESTING_URL", "https://www.investing.com/commodities/gold") +CSV_OUTPUT_PATH = os.getenv("CSV_OUTPUT_PATH", "./oi_data.csv") +TOP_N_STRIKES = int(os.getenv("TOP_N_STRIKES", "3")) +HEADLESS = os.getenv("HEADLESS", "false").lower() == "true" +TIMEOUT_SECONDS = int(os.getenv("TIMEOUT_SECONDS", "30")) +RETRY_ATTEMPTS = int(os.getenv("RETRY_ATTEMPTS", "3")) +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") +COOKIE_FILE = "./cookies.json" + +logging.basicConfig(level=getattr(logging, LOG_LEVEL)) +logger = logging.getLogger(__name__) + + +def save_cookies(context): + cookies = context.cookies() + with open(COOKIE_FILE, "w") as f: + json.dump(cookies, f) + logger.info("Cookies saved to file") + + +def load_cookies(context): + if os.path.exists(COOKIE_FILE): + with open(COOKIE_FILE, "r") as f: + cookies = json.load(f) + context.add_cookies(cookies) + logger.info("Cookies loaded from file") + return True + return False + + +def is_logged_in(page): + page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + return "login" not in page.url.lower() + + +def login_to_cme(page): + logger.info("Attempting to login to CME QuikStrike...") + + page.goto( + "https://www.cmegroup.com/account/login.html", timeout=TIMEOUT_SECONDS * 1000 + ) + + try: + page.fill('input[name="username"]', CME_USERNAME) + page.fill('input[name="password"]', CME_PASSWORD) + page.click('button[type="submit"]') + + page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + + if "login" in page.url.lower(): + logger.error("Login failed - still on login page") + page.screenshot(path="login_failed.png") + return False + + logger.info("Login successful") + page.screenshot(path="login_success.png") + return True + + except Exception as e: + logger.error(f"Login error: {e}") + page.screenshot(path="login_error.png") + return False + + +def navigate_to_oi_heatmap(page): + logger.info(f"Navigating to OI Heatmap: {PRODUCT_URL}") + page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + + +def extract_oi_data(page): + logger.info("Extracting OI data from Gold matrix table...") + + call_levels = [] + put_levels = [] + + table = page.locator("table.grid-thm").first + rows = table.locator("tbody tr").all() + + for row in rows: + try: + cells = row.locator("td").all() + if len(cells) < 3: + continue + + strike_cell = cells[0].text_content().strip() + if not strike_cell or not strike_cell.replace(".", "").isdigit(): + continue + + strike = float(strike_cell) + + cells_with_data = cells[2:] + + for i in range(0, len(cells_with_data), 2): + if i + 1 >= len(cells_with_data): + break + + call_cell = cells_with_data[i] + put_cell = cells_with_data[i + 1] + + call_text = call_cell.text_content().strip() + put_text = put_cell.text_content().strip() + + if call_text and call_text.replace(",", "").isdigit(): + call_oi = int(call_text.replace(",", "")) + call_levels.append( + {"Type": "CALL", "Strike": strike, "OI": call_oi} + ) + + if put_text and put_text.replace(",", "").isdigit(): + put_oi = int(put_text.replace(",", "")) + put_levels.append({"Type": "PUT", "Strike": strike, "OI": put_oi}) + + except Exception as e: + logger.warning(f"Error parsing row: {e}") + continue + + if not call_levels: + logger.warning("No CALL OI data extracted") + if not put_levels: + logger.warning("No PUT OI data extracted") + + call_df = ( + pd.DataFrame(call_levels).nlargest(TOP_N_STRIKES, "OI") + if call_levels + else pd.DataFrame() + ) + put_df = ( + pd.DataFrame(put_levels).nlargest(TOP_N_STRIKES, "OI") + if put_levels + else pd.DataFrame() + ) + + result_df = pd.concat([call_df, put_df], ignore_index=True) + + logger.info(f"Extracted {len(result_df)} OI levels") + return result_df + + +def scrape_investing_gold_price(page): + logger.info(f"Scraping gold price from: {INVESTING_URL}") + + try: + page.goto(INVESTING_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000) + + price_locator = page.locator('div[data-test="instrument-price-last"]') + + if price_locator.count() > 0: + price_text = price_locator.text_content().strip() + price_text = price_text.replace(",", "") + price = float(price_text) + logger.info(f"Extracted gold price: {price}") + return price + else: + logger.warning("Price element not found, trying alternative selector") + alt_locator = page.locator(".text-5xl\\/9") + if alt_locator.count() > 0: + price_text = alt_locator.text_content().strip() + price_text = price_text.replace(",", "") + price = float(price_text) + logger.info(f"Extracted gold price (alt): {price}") + return price + + logger.warning("Could not extract gold price") + return 0.0 + + except Exception as e: + logger.error(f"Error scraping gold price: {e}") + return 0.0 + + +def export_to_csv(df, future_price=0.0): + output_path = CSV_OUTPUT_PATH + + with open(output_path, "w") as f: + df.to_csv(f, index=False) + f.write("\n[Price]\n") + f.write(f"FuturePrice,{future_price}\n") + + logger.info(f"Exported OI data and price to {output_path}") + + +def run_scraper(): + if not CME_USERNAME or not CME_PASSWORD: + logger.error("Missing CME_USERNAME or CME_PASSWORD in .env file") + return + + future_price = 0.0 + + for attempt in range(RETRY_ATTEMPTS): + try: + with sync_playwright() as p: + browser = p.chromium.launch(headless=HEADLESS) + context = browser.new_context() + page = context.new_page() + + loaded_cookies = load_cookies(context) + page2 = context.new_page() + + if loaded_cookies and is_logged_in(page2): + logger.info("Using existing session (cookies)") + else: + logger.info("No valid session found, logging in...") + if not login_to_cme(page): + browser.close() + if attempt < RETRY_ATTEMPTS - 1: + logger.info( + f"Retrying... Attempt {attempt + 2}/{RETRY_ATTEMPTS}" + ) + continue + else: + logger.error("All login attempts failed") + return + save_cookies(context) + + navigate_to_oi_heatmap(page) + oi_data = extract_oi_data(page) + + if not oi_data.empty: + logger.info("Extracting gold price from investing.com...") + future_price = scrape_investing_gold_price(page) + + export_to_csv(oi_data, future_price) + else: + logger.warning("No OI data extracted") + + browser.close() + break + + except Exception as e: + logger.error(f"Scraper error (attempt {attempt + 1}): {e}") + if attempt < RETRY_ATTEMPTS - 1: + logger.info(f"Retrying... Attempt {attempt + 2}/{RETRY_ATTEMPTS}") + else: + logger.error("All attempts failed") + + +if __name__ == "__main__": + run_scraper() diff --git a/oi_scraper/requirements.txt b/oi_scraper/requirements.txt new file mode 100644 index 0000000..d29d1ce --- /dev/null +++ b/oi_scraper/requirements.txt @@ -0,0 +1,3 @@ +playwright>=1.40.0 +python-dotenv>=1.0.0 +pandas>=2.2.0 diff --git a/oi_scraper/run_scraper.bat b/oi_scraper/run_scraper.bat new file mode 100644 index 0000000..7356d73 --- /dev/null +++ b/oi_scraper/run_scraper.bat @@ -0,0 +1,47 @@ +@echo off +REM ========================================== +REM CME OI Scraper - Automatic Daily Runner +REM ========================================== + +REM Change to script directory +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +echo ========================================== +echo CME OI Scraper - Daily Update +echo ========================================== +echo Started at: %date% %time% +echo ========================================== >> scraper.log + +REM Run Python scraper +python main.py >> scraper.log 2>&1 + +REM Check if scraper succeeded +if %ERRORLEVEL% EQU 0 ( + echo [%date% %time%] Scraper completed successfully >> scraper.log + + REM Check if CSV file was created + if exist oi_data.csv ( + echo [%date% %time%] CSV file created successfully >> scraper.log + + REM Copy to MetaTrader 5 Files directory + REM Update this path to your actual MT5 directory + copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" + + if %ERRORLEVEL% EQU 0 ( + echo [%date% %time%] CSV copied to MT5 Files directory >> scraper.log + ) else ( + echo [%date% %time%] ERROR: Failed to copy CSV to MT5 directory >> scraper.log + ) + ) else ( + echo [%date% %time%] WARNING: oi_data.csv not found >> scraper.log + ) +) else ( + echo [%date% %time%] ERROR: Scraper failed with error code %ERRORLEVEL% >> scraper.log +) + +echo ========================================== +echo Completed at: %date% %time% +echo ========================================== + +REM Keep window open for 5 seconds to see any errors +timeout /t 5 diff --git a/oi_scraper/run_scraper.ps1 b/oi_scraper/run_scraper.ps1 new file mode 100644 index 0000000..28f3e11 --- /dev/null +++ b/oi_scraper/run_scraper.ps1 @@ -0,0 +1,77 @@ +# CME OI Scraper - PowerShell Script +# Copy this file to: run_scraper.ps1 + +# ========================================== +# Configuration +# ========================================== +$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper" +$logFile = "$scriptPath\scraper.log" +$csvFile = "$scriptPath\oi_data.csv" +$mt5Path = "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" + +# ========================================== +# Helper Functions +# ========================================== +function Write-Log { + param([string]$message) + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] $message" + Write-Output $logEntry | Add-Content $logFile + Write-Host $logEntry +} + +# ========================================== +# Main Script +# ========================================== + +# Navigate to script directory +cd $scriptPath + +Write-Log "==========================================" +Write-Log "CME OI Scraper - Daily Update" +Write-Log "==========================================" + +try { + # Run Python scraper + Write-Log "Starting Python scraper..." + & python main.py *>> $logFile 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -eq 0) { + Write-Log "Python scraper completed successfully" + + # Check if CSV was created + if (Test-Path $csvFile)) { + $fileInfo = Get-Item $csvFile + Write-Log "CSV file found (Last modified: $($fileInfo.LastWriteTime))" + + # Copy to MT5 directory + Write-Log "Copying CSV to MetaTrader 5 Files directory..." + + try { + Copy-Item -Path $csvFile -Destination $mt5Path -Force + Write-Log "CSV successfully copied to MT5 directory" + + # Verify copy + if (Test-Path $mt5Path)) { + Write-Log "Verified: MT5 CSV file exists" + } else { + Write-Log "ERROR: MT5 CSV file not found after copy" + } + } catch { + Write-Log "ERROR: Failed to copy to MT5 directory - $_" + } + } else { + Write-Log "WARNING: CSV file not found after scraper execution" + } + } else { + Write-Log "ERROR: Python scraper failed with exit code $exitCode" + } +} catch { + Write-Log "ERROR: Script failed - $($_.Exception.Message)" + exit 1 +} + +Write-Log "==========================================" +Write-Log "Script completed" +Write-Log "=========================================="