kunthawat/MeanRevisionEA
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(oi): improve data extraction and consolidate documentation

Browse Source 
- Fix MQL5 API usage in EA to use correct CopyRates and POSITION_TYPE enums
- Refactor scraper data extraction to use drop_duplicates for unique strikes
- Consolidate Windows setup guide into main README
- Add virtual environment batch files for easier setup and execution
- Simplify run_scraper.bat to focus on core execution
- Normalize lot calculation to use SymbolInfo.LotsStep()
This commit is contained in:
Kunthawat Greethong
2026-01-06 20:18:12 +07:00
parent 2e8e07ed17
commit b7c0e68fa8
8 changed files with 386 additions and 895 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
OI_OrderFlow_Absorption_XAUUSD.mq5
View File

@@ -358,15 +358,16 @@ void OnNewM1Bar(datetime newBarTime) {
} }
void CalculateVolumeEMAFromHistory() { void CalculateVolumeEMAFromHistory() {
double volumeArray[]; MqlRates rates[];
ArraySetAsSeries(volumeArray, true); ArraySetAsSeries(rates, true);
CopyVolume(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, volumeArray); int copied = CopyRates(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, rates);
if(copied < InpVolumeEmaPeriod + 1) return;
double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1); double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1);
double sum = 0; double sum = 0;
for(int i = 0; i < InpVolumeEmaPeriod; i++) { for(int i = 0; i < InpVolumeEmaPeriod; i++) {
sum += volumeArray[i]; sum += (double)rates[i].tick_volume;
} }
double avgVolume = sum / InpVolumeEmaPeriod; double avgVolume = sum / InpVolumeEmaPeriod;
@@ -378,13 +379,14 @@ void CalculateVolumeEMAFromHistory() {
} }
void DetectAbsorptionFromHistory() { void DetectAbsorptionFromHistory() {
double volumeArray[]; MqlRates rates[];
ArraySetAsSeries(volumeArray, true); ArraySetAsSeries(rates, true);
CopyVolume(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, volumeArray); int copied = CopyRates(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, rates);
if(copied < InpAbsorptionBars + 1) return;
double avgVolume = 0; double avgVolume = 0;
for(int i = 0; i < InpAbsorptionBars; i++) { for(int i = 0; i < InpAbsorptionBars; i++) {
avgVolume += volumeArray[i]; avgVolume += (double)rates[i].tick_volume;
} }
avgVolume /= InpAbsorptionBars; avgVolume /= InpAbsorptionBars;
@@ -394,11 +396,11 @@ void DetectAbsorptionFromHistory() {
int buyAbsorptionCount = 0; int buyAbsorptionCount = 0;
for(int i = 1; i <= InpAbsorptionBars; i++) { for(int i = 1; i <= InpAbsorptionBars; i++) {
double high = iHigh(_Symbol, PERIOD_M1, i); double high = rates[i].high;
double low = iLow(_Symbol, PERIOD_M1, i); double low = rates[i].low;
double close = iClose(_Symbol, PERIOD_M1, i); double close = rates[i].close;
double open = iOpen(_Symbol, PERIOD_M1, i); double open = rates[i].open;
int barVolume = (int)iVolume(_Symbol, PERIOD_M1, i); int barVolume = (int)rates[i].tick_volume;
double barRange = high - low; double barRange = high - low;
double barDrift = close - open; double barDrift = close - open;
@@ -415,7 +417,7 @@ void DetectAbsorptionFromHistory() {
} }
} }
int requiredBars = MathCeil(InpAbsorptionBars / 2.0); int requiredBars = (int)MathCeil(InpAbsorptionBars / 2.0);
if(sellAbsorptionCount >= requiredBars && IsPriceNearPutStrike()) { if(sellAbsorptionCount >= requiredBars && IsPriceNearPutStrike()) {
CurrentAbsorptionState = ABSORPTION_SELL; CurrentAbsorptionState = ABSORPTION_SELL;
@@ -679,7 +681,7 @@ bool IsAtSupport() {
} }
void ExecuteBuyTrade() { void ExecuteBuyTrade() {
double lotSize = CalculateLotSize(ORDER_TYPE_BUY); double lotSize = CalculateLotSize(POSITION_TYPE_BUY);
if(lotSize <= 0) return; if(lotSize <= 0) return;
double sl = 0, tp = 0; double sl = 0, tp = 0;
@@ -715,7 +717,7 @@ void ExecuteBuyTrade() {
} }
void ExecuteSellTrade() { void ExecuteSellTrade() {
double lotSize = CalculateLotSize(ORDER_TYPE_SELL); double lotSize = CalculateLotSize(POSITION_TYPE_SELL);
if(lotSize <= 0) return; if(lotSize <= 0) return;
double sl = 0, tp = 0; double sl = 0, tp = 0;
@@ -820,7 +822,7 @@ double CalculateLotSize(ENUM_POSITION_TYPE tradeType) {
} }
double NormalizeLot(double lot) { double NormalizeLot(double lot) {
double step = SymbolInfo.LotStep(); double step = SymbolInfo.LotsStep();
return MathFloor(lot / step) * step; return MathFloor(lot / step) * step;
} }
@@ -1037,6 +1039,9 @@ void CreateDashboard() {
} }
void UpdateDashboard() { void UpdateDashboard() {
MqlRates rates[];
int copied = CopyRates(_Symbol, PERIOD_M1, 0, 1, rates);
UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8); UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8);
UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8); UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8);
@@ -1063,7 +1068,7 @@ void UpdateDashboard() {
UpdateLabel("DB_Absorption", 20, 105, "Absorption: " + absorptionText, absorptionColor, 8); UpdateLabel("DB_Absorption", 20, 105, "Absorption: " + absorptionText, absorptionColor, 8);
int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (int)Volume(0); int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (copied > 0 ? (int)rates[0].tick_volume : 0);
UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) + UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) +
" (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8); " (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8);

347
oi_scraper/README.md
View File

@@ -1,179 +1,268 @@
# CME OI Scraper # CME OI Scraper
Python scraper to pull Open Interest data from CME Group QuikStrike and current gold price from investing.com. Python scraper that extracts Open Interest data from CME Group QuikStrike and current gold price from investing.com.
## What It Extracts ## What It Extracts
1. **OI Levels (from CME QuikStrike):** 1. **OI Levels (from CME QuikStrike):**
- Top 3 CALL strikes by OI volume - Top 3 CALL strikes by OI volume (unique strikes)
- Top 3 PUT strikes by OI volume - Top 3 PUT strikes by OI volume (unique strikes)
2. **Gold Price (from investing.com):** 2. **Gold Price (from investing.com):**
- Current gold futures price (e.g., 4345.50) - Current gold futures price (e.g., 4476.50)
## Prerequisites ## Prerequisites
- Python 3.9+ - Python 3.9 or higher
- CME Group QuikStrike account with login credentials - CME Group QuikStrike account (free registration at https://www.cmegroup.com)
- Windows 10/11 (for batch files) or Linux/macOS
## Installation ## Quick Start
1. Copy environment variables: ### Windows
```bash
cp .env.example .env
```
2. Edit `.env` and add your CME credentials: 1. **Run one-time setup:**
```bash ```cmd
CME_USERNAME=your_username cd C:\Path\To\oi_scraper
CME_PASSWORD=your_password setup_env.bat
``` ```
3. Install dependencies: 2. **Run the scraper:**
```bash ```cmd
pip install -r requirements.txt run_with_venv.bat
playwright install chromium ```
```
## Usage ### Linux/macOS
### Basic Scraping 1. **Setup:**
```bash
cd /path/to/oi_scraper
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
playwright install chromium
```
```bash 2. **Run:**
python main.py ```bash
``` source venv/bin/activate
python main.py
This will: ```
- Login to CME QuikStrike
- Navigate to OI Heatmap
- Extract top 3 CALL and PUT strikes by OI volume
- Scrape current gold price from investing.com
- Export to `oi_data.csv`
### Session Persistence
The scraper automatically saves your login session to `cookies.json`. This means:
- **First run**: Logs in with your credentials, saves cookies
- **Subsequent runs**: Uses saved cookies if session is still valid
- **Session expired**: Automatically logs in again and saves new cookies
Benefits for scheduled runs:
- Faster execution (skips login when session is valid)
- Reduces login attempts to CME servers
- CME sessions typically last several days/weeks
To force a fresh login, delete `cookies.json`:
```bash
rm cookies.json
```
### Output Format
The CSV output is compatible with the EA's `LoadOIFromCSV()` and `LoadFuturePriceFromCSV()` functions:
```csv
Type,Strike,OI
CALL,4345,155398
CALL,4350,229137
CALL,4360,90649
PUT,4300,227936
PUT,4290,270135
PUT,4280,65839
[Price]
FuturePrice,4345.50
```
**Note:** The `[Price]` section contains the current gold futures price scraped from investing.com. The EA reads this value for Delta calculation.
## Configuration ## Configuration
Edit `.env` to customize: ### Edit `.env` File
- `PRODUCT_URL` - QuikStrike product page URL (requires login) Copy and edit the environment file:
- `CME_LOGIN_URL` - CME login page URL (default: SSO URL)
- `TOP_N_STRIKES` - Number of top strikes to export (default: 3)
- `HEADLESS` - Run browser in headless mode (default: false for debugging)
- `CSV_OUTPUT_PATH` - Output CSV file path
- `TIMEOUT_SECONDS` - Page load timeout
### Available Products ```cmd
copy .env.example .env
**Gold (XAUUSD/COMEX Gold - OG|GC):** notepad .env
```
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
``` ```
**Silver:** Required settings:
``` ```env
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool CME_USERNAME=your_cme_username
CME_PASSWORD=your_cme_password
``` ```
**SOFR (3M SOFR):** Optional settings:
``` ```env
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool # Number of top strikes to export (default: 3)
TOP_N_STRIKES=3
# Run browser without window (default: false)
HEADLESS=false
# Page timeout in seconds (default: 30)
TIMEOUT_SECONDS=30
# Output CSV path
CSV_OUTPUT_PATH=./oi_data.csv
# Logging level: DEBUG, INFO, WARNING, ERROR
LOG_LEVEL=INFO
``` ```
**Note:** You must be logged in to access QuikStrike data. The scraper will automatically login using credentials from `.env`. ## Output Format
The scraper exports to `oi_data.csv`:
```csv
Type,Strike,OI
CALL,4375.0,147
CALL,4450.0,173
CALL,4500.0,176
PUT,4435.0,49
PUT,4400.0,102
PUT,4515.0,150
[Price]
FuturePrice,4467.8
```
The `[Price]` section contains the current gold futures price scraped from investing.com.
## Session Persistence
The scraper saves login sessions to `cookies.json`:
- **First run:** Logs in with credentials, saves cookies
- **Subsequent runs:** Uses saved cookies if session is valid
- **Session expired:** Automatically re-logs in and saves new cookies
This makes scheduled runs faster and reduces login attempts to CME servers.
To force a fresh login:
```cmd
del cookies.json
```
## Integration with EA ## Integration with EA
The EA reads OI data from CSV when `InpOISource = OI_SOURCE_CSV_FILE`. The EA reads OI data from CSV when configured:
```mql5
input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE;
```
Place the generated `oi_data.csv` in MetaTrader's `MQL5/Files` directory. Copy `oi_data.csv` to your MT5 `MQL5/Files` directory:
```
C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\Common\MQL5\Files\oi_data.csv
```
## Scheduling ## Automatic Daily Scheduling
Use cron or Windows Task Scheduler to run periodically: ### Windows Task Scheduler
1. **Create scheduled task:**
- Open Task Scheduler (`taskschd.msc`)
- Click "Create Task"
2. **Configure General tab:**
- Name: `CME OI Scraper - Daily`
- ✅ Run whether user is logged on or not
- ✅ Run with highest privileges
3. **Configure Triggers tab:**
- New → On a schedule → Daily
- Start time: 9:00 AM (or your preferred time)
- ✅ Enabled
4. **Configure Actions tab:**
- Action: Start a program
- Program/script:
```
C:\Path\To\oi_scraper\run_scheduled.bat
```
- Start in:
```
C:\Path\To\oi_scraper
```
5. **Click OK to save**
### Linux/macOS (cron)
```bash ```bash
# Run every hour # Edit crontab
0 * * * * cd /path/to/oi_scraper && python main.py crontab -e
# Add line to run every day at 9 AM
0 9 * * * cd /path/to/oi_scraper && /path/to/venv/bin/python main.py
``` ```
## Batch Files Reference
| File | Purpose |
|------|---------|
| `setup_env.bat` | One-time setup (creates virtual environment) |
| `run_with_venv.bat` | Manual run with visible window |
| `run_scheduled.bat` | For Task Scheduler (no window, no pause) |
## Troubleshooting ## Troubleshooting
**Login fails:** ### Module Not Found Errors
**Error:** `ModuleNotFoundError: No module named 'playwright'`
**Solution:**
```cmd
run_with_venv.bat
```
The virtual environment ensures all dependencies are isolated.
### Login Fails
- Verify credentials in `.env` - Verify credentials in `.env`
- Check if CME requires 2FA - Check if CME requires 2FA (manual intervention needed)
- Set `HEADLESS=false` to see what's happening - Set `HEADLESS=false` to see browser activity
- Check screenshots: `login_failed.png`, `login_error.png`, `login_success.png` - Check screenshots: `login_failed.png`, `login_error.png`
**No data extracted:** ### No Data Extracted
- Check if table structure changed
- Increase `TIMEOUT_SECONDS`
- Check logs for detailed errors
- Screenshot saved as `login_debug.png` or `login_failed.png`
**Login page selectors changed:** - Check if CME table structure changed
- If the scraper can't find username/password inputs, CME may have updated their login page - Increase `TIMEOUT_SECONDS=60` in `.env`
- Update the selectors in `login_to_cme()` function in `main.py`: - Check logs for errors
```python - Screenshot saved as `login_debug.png`
# Example: update to match current CME login form
page.fill('input[id="username"]', CME_USERNAME)
page.fill('input[id="password"]', CME_PASSWORD)
page.click('button[type="submit"]')
```
**Browser issues:** ### Browser Issues
- Install Chromium dependencies: `playwright install chromium`
- Try different browser: Change `p.chromium.launch()` to `p.firefox.launch()` ```cmd
# Reinstall Chromium
python -m playwright install chromium
```
### Session Expires Frequently
Delete cookies to force fresh login:
```cmd
del cookies.json
```
### Check Python Path Issues (Windows)
```cmd
# Check which Python is being used
where python
# Use Python launcher
py -3 main.py
# Or use the virtual environment
run_with_venv.bat
```
## Finding Product IDs
To scrape other instruments (Silver, Crude Oil, etc.):
1. Visit CME QuikStrike OI Heatmap
2. Login to your CME account
3. Select a product from the dropdown
4. The URL updates with the `pid` parameter
5. Note: This scraper is configured for Gold by default
## Notes ## Notes
- The scraper targets the OI Heatmap table structure - Targets the OI Heatmap table structure
- Only exports top N strikes by OI volume - Exports top N unique strikes by OI volume
- Login session is not persisted (login each run) - Uses session cookies for faster subsequent runs
- Cookies could be saved for faster subsequent runs - CME sessions typically last several days to weeks
- Virtual environment recommended to avoid Python path conflicts
### Finding Product IDs ## Files
To find product IDs for other instruments: ```
1. Visit https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html oi_scraper/
2. Login to your CME account ├── main.py # Main scraper script
3. Select a product from the "Products" menu ├── requirements.txt # Python dependencies
4. The URL will update with the `pid` parameter ├── .env.example # Environment template
5. Copy that URL to your `.env` file ├── .env # Your credentials (create from example)
├── setup_env.bat # Windows: Create virtual environment
Example: `https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html?pid=40` (Gold) ├── run_with_venv.bat # Windows: Manual run
├── run_scheduled.bat # Windows: Task Scheduler run
├── oi_data.csv # Output file (generated)
├── cookies.json # Session cookies (generated)
└── scraper.log # Log file (generated)
```

658
oi_scraper/WINDOWS_SETUP.md
View File

@@ -1,658 +0,0 @@
# CME OI Scraper - Windows Setup Guide
Complete guide for setting up and running the CME OI scraper on Windows with automatic daily updates.
## Table of Contents
- [Prerequisites](#prerequisites)
- [Installation](#installation)
- [Configuration](#configuration)
- [Manual Testing](#manual-testing)
- [Automatic Daily Updates](#automatic-daily-updates)
- [MetaTrader 5 Integration](#metatrader-5-integration)
- [Troubleshooting](#troubleshooting)
---
## Prerequisites
### Required Software
1. **Python 3.9 or higher**
- Download: https://www.python.org/downloads/
- During installation: ✅ Check "Add Python to PATH"
2. **CME Group QuikStrike Account**
- Free account required: https://www.cmegroup.com/
- Register for QuikStrike access
- Save your username and password
3. **MetaTrader 5** (for EA integration)
- Download: https://www.metatrader5.com/
- Install on your Windows machine
### Verify Python Installation
```cmd
python --version
```
Expected output: `Python 3.9.x` or higher
If not found, install Python or use `py` or `python3` commands.
---
## Installation
### Step 1: Navigate to Scraper Directory
Open Command Prompt (cmd) and navigate:
```cmd
cd C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
```
Replace `YourUsername` with your actual Windows username.
### Step 2: Create Environment File
```cmd
copy .env.example .env
```
### Step 3: Edit .env File
Open `.env` with Notepad:
```cmd
notepad .env
```
Update with your credentials:
```env
# CME Group QuikStrike Login Credentials
CME_USERNAME=your_actual_username_here
CME_PASSWORD=your_actual_password_here
# Product Configuration (Gold)
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
# Output Settings
CSV_OUTPUT_PATH=./oi_data.csv
TOP_N_STRIKES=3
# Scraping Settings
HEADLESS=false
TIMEOUT_SECONDS=30
RETRY_ATTEMPTS=3
# Logging
LOG_LEVEL=INFO
```
**Save and close** (Ctrl+S, then Alt+F4).
### Step 4: Install Python Dependencies
```cmd
pip install -r requirements.txt
```
Expected output: Successfully installed playwright, python-dotenv, pandas
### Step 5: Install Playwright Browser
```cmd
playwright install chromium
```
Expected output: Downloading Chromium... [progress bar]
---
## Configuration
### Available Products
**Gold (XAUUSD/COMEX Gold - OG|GC):**
```env
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
```
**Silver:**
```env
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool
```
**SOFR (3M SOFR):**
```env
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool
```
### Configuration Options
| Setting | Description | Default |
|----------|-------------|---------|
| `TOP_N_STRIKES` | Number of top strikes to export | 3 |
| `HEADLESS` | Run browser without window (true/false) | false |
| `TIMEOUT_SECONDS` | Page load timeout in seconds | 30 |
| `CSV_OUTPUT_PATH` | Output CSV file path | ./oi_data.csv |
| `LOG_LEVEL` | DEBUG, INFO, WARNING, ERROR | INFO |
---
## Manual Testing
### Run Scraper Manually
```cmd
python main.py
```
Expected output:
```
INFO:__main__:Cookies loaded from file
INFO:__main__:Using existing session (cookies)
INFO:__main__:Navigating to OI Heatmap: https://...
INFO:__main__:Extracting OI data from Gold matrix table...
INFO:__main__:Extracted 6 OI levels
INFO:__main__:Exported OI data to ./oi_data.csv
```
### Check Output
**1. Verify CSV created:**
```cmd
dir oi_data.csv
```
**2. View CSV content:**
```cmd
notepad oi_data.csv
```
Expected format:
```csv
Type,Strike,OI
CALL,4400,6193
CALL,4300,3826
CALL,4350,1983
PUT,4400,5559
PUT,4300,2988
PUT,4350,1214
```
### Check Logs
```cmd
type scraper.log
```
Or view in Notepad:
```cmd
notepad scraper.log
```
---
## Automatic Daily Updates
### Option 1: Windows Task Scheduler (Recommended)
#### Step 1: Create Batch File Wrapper
Create `run_scraper.bat` in the oi_scraper directory:
```cmd
@echo off
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
echo Starting CME OI Scraper at %date% %time% >> scraper.log
echo ---------------------------------------- >> scraper.log
python main.py >> scraper.log 2>&1
if %ERRORLEVEL% EQU 0 (
echo %date% %time%: Scraper completed successfully >> scraper.log
) else (
echo %date% %time%: Scraper failed with error %ERRORLEVEL% >> scraper.log
)
echo ---------------------------------------- >> scraper.log
```
Replace `YourUsername` with your actual username.
#### Step 2: Open Task Scheduler
Press `Win + R`, type `taskschd.msc`, press Enter
Or: Start → Windows Administrative Tools → Task Scheduler
#### Step 3: Create Task
1. Click **"Create Basic Task"** on right sidebar
2. **Name:** `CME OI Scraper - Daily`
3. **Description:** `Update OI data from CME QuikStrike every day at 9 AM`
4. Click **Next**
#### Step 4: Set Trigger
1. **Trigger:** Select "Daily"
2. **Start date:** Today's date
3. **Start time:** 9:00:00 AM (or your preferred time)
4. Click **Next**
#### Step 5: Set Action
1. **Action:** Select "Start a program"
2. **Program/script:**
```
C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.bat
```
3. **Start in (optional):**
```
C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
```
4. Click **Next**
#### Step 6: Finish
1. Review settings
2. Check "Open the Properties dialog for this task when I click Finish"
3. Click **Finish**
#### Step 7: Configure Advanced Settings (Optional)
In the Properties dialog:
- **General tab:**
- ✅ Run whether user is logged on or not
- ✅ Do not store password (if using Windows authentication)
- ✅ Run with highest privileges
- **Conditions tab:**
- ✅ Start the task only if the computer is on AC power
- ✅ Stop if the computer switches to battery power
- ✅ Wake the computer to run this task
- **Settings tab:**
- ✅ Allow task to be run on demand
- ❌ Stop the task if it runs longer than: 30 minutes
- ✅ If the task fails, restart every: 5 minutes (up to 3 times)
Click **OK** to save settings.
#### Step 8: Test Task
1. In Task Scheduler, find "CME OI Scraper - Daily"
2. Right-click → **Run**
3. Check `scraper.log` after a minute:
```cmd
type scraper.log
```
---
### Option 2: PowerShell Script (Advanced)
#### Step 1: Create PowerShell Script
Save as `run_scraper.ps1`:
```powershell
# Script configuration
$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper"
$logFile = "$scriptPath\scraper.log"
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
# Navigate to script directory
cd $scriptPath
try {
# Run Python scraper
Write-Output "$timestamp: Starting CME OI Scraper" | Add-Content $logFile
& python main.py *>> $logFile 2>&1
# Check if CSV was created
if (Test-Path "oi_data.csv") {
$fileInfo = Get-Item "oi_data.csv"
Write-Output "$timestamp: Scraper completed successfully (CSV updated: $($fileInfo.LastWriteTime))" | Add-Content $logFile
} else {
Write-Output "$timestamp: WARNING - CSV file not created" | Add-Content $logFile
}
} catch {
$errorMsg = $_.Exception.Message
Write-Output "$timestamp: ERROR - $errorMsg" | Add-Content $logFile
exit 1
}
```
#### Step 2: Update Task Scheduler to Use PowerShell
Same steps as Option 1, but:
- **Program/script:** `powershell.exe`
- **Add arguments:**
```
-ExecutionPolicy Bypass -File "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.ps1"
```
---
## MetaTrader 5 Integration
### Find MT5 Files Directory
MT5 data directory location:
```
C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Terminal_ID]\MQL5\Files\
```
**To find your Terminal_ID:**
1. Open MT5
2. Click **File** → **Open Data Folder**
3. Navigate to `Terminal\[Your_Terminal_ID]\MQL5\Files\`
### Update Batch File to Copy to MT5
Edit `run_scraper.bat`:
```cmd
@echo off
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
echo Starting CME OI Scraper at %date% %time% >> scraper.log
echo ---------------------------------------- >> scraper.log
python main.py >> scraper.log 2>&1
if %ERRORLEVEL% EQU 0 (
if exist oi_data.csv (
echo Copying OI data to MT5... >> scraper.log
copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
if %ERRORLEVEL% EQU 0 (
echo %date% %time%: Scraper completed - OI data copied to MT5 >> scraper.log
) else (
echo %date% %time%: ERROR - Failed to copy to MT5 >> scraper.log
)
) else (
echo %date% %time%: ERROR - oi_data.csv not found >> scraper.log
)
) else (
echo %date% %time%: ERROR - Scraper failed with error %ERRORLEVEL% >> scraper.log
)
echo ---------------------------------------- >> scraper.log
```
Replace `[Your_Terminal_ID]` with your actual MT5 terminal ID.
### Update EA Configuration
In your EA (`OI_MeanReversion_Pro_XAUUSD_A.mq5`), set:
```mql5
input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE; // Load from CSV file
```
The EA will automatically read `oi_data.csv` from its Files directory.
---
## Troubleshooting
### Python Not Found
**Error:** `'python' is not recognized as an internal or external command`
**Solutions:**
1. Use full path to Python:
```cmd
C:\Users\YourUsername\AppData\Local\Programs\Python\Python312\python.exe main.py
```
2. Use `py` launcher:
```cmd
py main.py
```
3. Reinstall Python with "Add to PATH" option
### Module Import Errors
**Error:** `ModuleNotFoundError: No module named 'playwright'`
**Solution:**
```cmd
pip install -r requirements.txt
```
### Login Fails
**Error:** `Login failed - still on login page`
**Solutions:**
1. Check credentials in `.env` file:
```cmd
notepad .env
```
2. Check login screenshots:
- `login_failed.png` - Shows login page
- `login_error.png` - Shows error during login
- `login_success.png` - Confirms successful login
3. Manually test login at: https://www.cmegroup.com/
4. Check if 2FA is required (CME may require additional authentication)
### No Data Extracted
**Warning:** `No CALL OI data extracted` or `No PUT OI data extracted`
**Solutions:**
1. Check if you're logged in:
- Delete `cookies.json` to force fresh login
- Run scraper manually with `HEADLESS=false` in `.env`
2. Check if page structure changed:
- View screenshots to see actual page content
- Check if Gold product URL is correct
3. Increase timeout:
```env
TIMEOUT_SECONDS=60
```
### Task Not Running
**Issue:** Task Scheduler doesn't execute the task
**Solutions:**
1. Check task history:
- Task Scheduler → Right-click task → Properties → History tab
- Look for errors in the log
2. Test manually:
- Right-click task → Run
- Check `scraper.log` for output
3. Check account permissions:
- Ensure task is set to run with your Windows account
- Check "Run whether user is logged on or not"
4. Check Windows Event Viewer:
- Event Viewer → Windows Logs → Application
- Look for Task Scheduler errors
### Session Expiration
**Issue:** Session expires after some time
**Solution:**
The scraper will automatically re-login when cookies expire. No manual action needed.
To force fresh login:
```cmd
del cookies.json
```
### Check Logs
**View recent logs:**
```cmd
type scraper.log | more
```
**View last 20 lines:**
```cmd
powershell "Get-Content scraper.log -Tail 20"
```
**Search for errors:**
```cmd
findstr /C:"ERROR" scraper.log
```
### Verify CSV Output
**Check if CSV is valid:**
```cmd
python -c "import pandas as pd; print(pd.read_csv('oi_data.csv'))"
```
**Check file size:**
```cmd
dir oi_data.csv
```
---
## Advanced Options
### Run Multiple Times Per Day
**Edit Task Scheduler Trigger:**
1. Open task properties → Triggers tab
2. Edit existing trigger → Click "New" to add additional
3. Set different times:
- 9:00 AM
- 12:00 PM
- 3:00 PM
- 6:00 PM
### Run on Market Days Only
**Create separate batch file:**
```cmd
@echo off
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
REM Check if today is weekday (1=Monday, 5=Friday)
for /f "skip=1 tokens=*" %%a in ('wmic path win32_localtime get dayofweek /value') do set DAY=%%a
if %DAY% LSS 1 goto END
if %DAY% GTR 5 goto END
REM Run scraper
python main.py >> scraper.log 2>&1
:END
```
### Email Notifications
**Use PowerShell to send email on completion:**
```powershell
# Add to run_scraper.ps1 at the end
$smtpServer = "smtp.gmail.com"
$smtpPort = 587
$smtpUser = "your_email@gmail.com"
$smtpPass = "your_password"
$from = "CME OI Scraper <your_email@gmail.com>"
$to = "your_email@gmail.com"
$subject = "CME OI Scraper - %date%"
if ($errorOccurred) {
$body = "CME OI Scraper failed. Check logs for details."
} else {
$body = "CME OI Scraper completed successfully.`n`nUpdated files:`n- oi_data.csv"
}
$message = New-Object System.Net.Mail.MailMessage $from, $to
$message.Subject = $subject
$message.Body = $body
$smtp = New-Object System.Net.Mail.SmtpClient $smtpServer, $smtpPort
$smtp.EnableSsl = $true
$smtp.Credentials = New-Object System.Net.NetworkCredential $smtpUser, $smtpPass
$smtp.Send($message)
```
---
## Summary
**Quick Start Checklist:**
- [ ] Python 3.9+ installed
- [ ] CME QuikStrike account created
- [ ] `.env` file configured with credentials
- [ ] Dependencies installed (`pip install -r requirements.txt`)
- [ ] Playwright browser installed (`playwright install chromium`)
- [ ] Manual test successful (`python main.py`)
- [ ] `oi_data.csv` created and valid
- [ ] Task Scheduler task created
- [ ] Task tested manually
- [ ] CSV copied to MT5 Files directory
- [ ] EA configured to use CSV file
**Daily Workflow:**
1. Task Scheduler runs at 9:00 AM
2. Batch file executes Python scraper
3. Scraper logs in with saved cookies (or fresh login)
4. OI data extracted and saved to `oi_data.csv`
5. CSV copied to MT5 Files directory
6. EA reads updated OI data
7. EA uses new OI levels for trading
---
## Support
For issues or questions:
1. Check `scraper.log` for detailed error messages
2. Review screenshots (login_failed.png, login_error.png)
3. Verify `.env` configuration
4. Test manually without Task Scheduler
5. Check Windows Event Viewer for system errors
---
**Last Updated:** January 4, 2026
**Version:** 1.0
**Platform:** Windows 10/11

25
oi_scraper/main.py
View File

@@ -1,3 +1,11 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
CME OI Scraper - Extracts Open Interest data from CME QuikStrike and gold price from investing.com
Usage: python main.py
Requires: pip install -r requirements.txt
"""
import os import os
import logging import logging
import json import json
@@ -212,19 +220,24 @@ def extract_oi_data(page):
if call_levels: if call_levels:
call_df = pd.DataFrame(call_levels) call_df = pd.DataFrame(call_levels)
call_df = call_df.groupby("Strike", as_index=False).agg({"OI": "max"}) call_df = call_df.drop_duplicates(subset="Strike", keep="first")
call_df = call_df.nlargest(TOP_N_STRIKES, "OI") call_df = call_df.sort_values("OI")
call_df = call_df.tail(TOP_N_STRIKES)
call_df["Type"] = "CALL"
else: else:
call_df = pd.DataFrame() call_df = pd.DataFrame()
if put_levels: if put_levels:
put_df = pd.DataFrame(put_levels) put_df = pd.DataFrame(put_levels)
put_df = put_df.groupby("Strike", as_index=False).agg({"OI": "max"}) put_df = put_df.drop_duplicates(subset="Strike", keep="first")
put_df = put_df.nlargest(TOP_N_STRIKES, "OI") put_df = put_df.sort_values("OI")
put_df = put_df.tail(TOP_N_STRIKES)
put_df["Type"] = "PUT"
else: else:
put_df = pd.DataFrame() put_df = pd.DataFrame()
result_df = pd.concat([call_df, put_df], ignore_index=True) result_df = pd.concat([call_df, put_df])
result_df = result_df[["Type", "Strike", "OI"]]
logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted") logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted")
return result_df return result_df
@@ -327,7 +340,7 @@ def run_scraper():
oi_data = extract_oi_data(page) oi_data = extract_oi_data(page)
save_cookies(context) save_cookies(context)
if not oi_data.empty: if len(oi_data) > 0:
logger.info("Extracting gold price from investing.com...") logger.info("Extracting gold price from investing.com...")
future_price = scrape_investing_gold_price(page) future_price = scrape_investing_gold_price(page)
logger.info(f"Gold price extracted: {future_price}") logger.info(f"Gold price extracted: {future_price}")

13
oi_scraper/run_scheduled.bat Normal file
View File

@@ -0,0 +1,13 @@
@echo off
REM ==========================================
REM CME OI Scraper - Scheduled Task Version
REM For use with Windows Task Scheduler
REM ==========================================
REM Navigate to script directory
cd /d %~dp0
REM Activate virtual environment and run scraper (no pause)
call venv\Scripts\activate.bat
python main.py
exit %ERRORLEVEL%

46
oi_scraper/run_scraper.bat
View File

@@ -1,47 +1,21 @@
@echo off @echo off
REM ========================================== REM ==========================================
REM CME OI Scraper - Automatic Daily Runner REM CME OI Scraper - Run with Virtual Environment
REM ========================================== REM ==========================================
REM Change to script directory REM Navigate to script directory
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper cd /d %~dp0
echo ========================================== echo ==========================================
echo CME OI Scraper - Daily Update echo CME OI Scraper
echo ========================================== echo ==========================================
echo Started at: %date% %time%
echo ========================================== >> scraper.log REM Activate virtual environment
call venv\Scripts\activate.bat
REM Run Python scraper REM Run Python scraper
python main.py >> scraper.log 2>&1 python main.py
REM Check if scraper succeeded REM Pause for 5 seconds if running manually (not scheduled)
if %ERRORLEVEL% EQU 0 ( if "%1"=="--scheduled" goto :eof
echo [%date% %time%] Scraper completed successfully >> scraper.log
REM Check if CSV file was created
if exist oi_data.csv (
echo [%date% %time%] CSV file created successfully >> scraper.log
REM Copy to MetaTrader 5 Files directory
REM Update this path to your actual MT5 directory
copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
if %ERRORLEVEL% EQU 0 (
echo [%date% %time%] CSV copied to MT5 Files directory >> scraper.log
) else (
echo [%date% %time%] ERROR: Failed to copy CSV to MT5 directory >> scraper.log
)
) else (
echo [%date% %time%] WARNING: oi_data.csv not found >> scraper.log
)
) else (
echo [%date% %time%] ERROR: Scraper failed with error code %ERRORLEVEL% >> scraper.log
)
echo ==========================================
echo Completed at: %date% %time%
echo ==========================================
REM Keep window open for 5 seconds to see any errors
timeout /t 5 timeout /t 5

21
oi_scraper/run_with_venv.bat Normal file
View File

@@ -0,0 +1,21 @@
@echo off
REM ==========================================
REM CME OI Scraper - Manual Run with Virtual Environment
REM ==========================================
REM Navigate to script directory
cd /d %~dp0
echo ==========================================
echo CME OI Scraper - Manual Run
echo ==========================================
REM Activate virtual environment
call venv\Scripts\activate.bat
REM Run Python scraper
python main.py
echo.
echo Scraper completed. Check oi_data.csv for results.
timeout /t 5

34
oi_scraper/setup_env.bat Normal file
View File

@@ -0,0 +1,34 @@
@echo off
REM ==========================================
REM CME OI Scraper - Virtual Environment Setup
REM ==========================================
echo ==========================================
echo Setting up Python Virtual Environment
echo ==========================================
REM Navigate to script directory
cd /d %~dp0
REM Create virtual environment
echo Creating virtual environment...
py -3 -m venv venv
REM Activate virtual environment and install dependencies
echo Installing dependencies...
call venv\Scripts\activate.bat
pip install --upgrade pip
pip install -r requirements.txt
REM Install playwright browser
echo Installing Playwright browser...
python -m playwright install chromium
echo ==========================================
echo Setup Complete!
echo ==========================================
echo.
echo To run the scraper, use: run_with_venv.bat
echo.
pause