refactor(oi): improve data extraction and consolidate documentation
- Fix MQL5 API usage in EA to use correct CopyRates and POSITION_TYPE enums - Refactor scraper data extraction to use drop_duplicates for unique strikes - Consolidate Windows setup guide into main README - Add virtual environment batch files for easier setup and execution - Simplify run_scraper.bat to focus on core execution - Normalize lot calculation to use SymbolInfo.LotsStep()
This commit is contained in:
Kunthawat Greethong
@@ -358,64 +358,66 @@ void OnNewM1Bar(datetime newBarTime) {
|
||||
}
|
||||
|
||||
void CalculateVolumeEMAFromHistory() {
|
||||
double volumeArray[];
|
||||
ArraySetAsSeries(volumeArray, true);
|
||||
CopyVolume(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, volumeArray);
|
||||
|
||||
double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1);
|
||||
double sum = 0;
|
||||
|
||||
for(int i = 0; i < InpVolumeEmaPeriod; i++) {
|
||||
sum += volumeArray[i];
|
||||
}
|
||||
double avgVolume = sum / InpVolumeEmaPeriod;
|
||||
|
||||
if(VolumeEmaValue == 0) {
|
||||
VolumeEmaValue = avgVolume;
|
||||
} else {
|
||||
VolumeEmaValue = emaAlpha * avgVolume + (1 - emaAlpha) * VolumeEmaValue;
|
||||
}
|
||||
MqlRates rates[];
|
||||
ArraySetAsSeries(rates, true);
|
||||
int copied = CopyRates(_Symbol, PERIOD_M1, 1, InpVolumeEmaPeriod + 1, rates);
|
||||
if(copied < InpVolumeEmaPeriod + 1) return;
|
||||
|
||||
double emaAlpha = 2.0 / (InpVolumeEmaPeriod + 1);
|
||||
double sum = 0;
|
||||
|
||||
for(int i = 0; i < InpVolumeEmaPeriod; i++) {
|
||||
sum += (double)rates[i].tick_volume;
|
||||
}
|
||||
double avgVolume = sum / InpVolumeEmaPeriod;
|
||||
|
||||
if(VolumeEmaValue == 0) {
|
||||
VolumeEmaValue = avgVolume;
|
||||
} else {
|
||||
VolumeEmaValue = emaAlpha * avgVolume + (1 - emaAlpha) * VolumeEmaValue;
|
||||
}
|
||||
}
|
||||
|
||||
void DetectAbsorptionFromHistory() {
|
||||
double volumeArray[];
|
||||
ArraySetAsSeries(volumeArray, true);
|
||||
CopyVolume(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, volumeArray);
|
||||
|
||||
double avgVolume = 0;
|
||||
for(int i = 0; i < InpAbsorptionBars; i++) {
|
||||
avgVolume += volumeArray[i];
|
||||
}
|
||||
avgVolume /= InpAbsorptionBars;
|
||||
|
||||
double volumeThreshold = avgVolume * InpMinVolumeMultiplier;
|
||||
|
||||
int sellAbsorptionCount = 0;
|
||||
int buyAbsorptionCount = 0;
|
||||
|
||||
for(int i = 1; i <= InpAbsorptionBars; i++) {
|
||||
double high = iHigh(_Symbol, PERIOD_M1, i);
|
||||
double low = iLow(_Symbol, PERIOD_M1, i);
|
||||
double close = iClose(_Symbol, PERIOD_M1, i);
|
||||
double open = iOpen(_Symbol, PERIOD_M1, i);
|
||||
int barVolume = (int)iVolume(_Symbol, PERIOD_M1, i);
|
||||
|
||||
double barRange = high - low;
|
||||
double barDrift = close - open;
|
||||
|
||||
bool highVolume = barVolume > volumeThreshold;
|
||||
bool lowDrift = barRange < InpMaxPriceDriftPoints * _Point;
|
||||
|
||||
if(highVolume && lowDrift && barDrift < 0) {
|
||||
sellAbsorptionCount++;
|
||||
}
|
||||
|
||||
if(highVolume && lowDrift && barDrift > 0) {
|
||||
buyAbsorptionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
int requiredBars = MathCeil(InpAbsorptionBars / 2.0);
|
||||
MqlRates rates[];
|
||||
ArraySetAsSeries(rates, true);
|
||||
int copied = CopyRates(_Symbol, PERIOD_M1, 1, InpAbsorptionBars + 1, rates);
|
||||
if(copied < InpAbsorptionBars + 1) return;
|
||||
|
||||
double avgVolume = 0;
|
||||
for(int i = 0; i < InpAbsorptionBars; i++) {
|
||||
avgVolume += (double)rates[i].tick_volume;
|
||||
}
|
||||
avgVolume /= InpAbsorptionBars;
|
||||
|
||||
double volumeThreshold = avgVolume * InpMinVolumeMultiplier;
|
||||
|
||||
int sellAbsorptionCount = 0;
|
||||
int buyAbsorptionCount = 0;
|
||||
|
||||
for(int i = 1; i <= InpAbsorptionBars; i++) {
|
||||
double high = rates[i].high;
|
||||
double low = rates[i].low;
|
||||
double close = rates[i].close;
|
||||
double open = rates[i].open;
|
||||
int barVolume = (int)rates[i].tick_volume;
|
||||
|
||||
double barRange = high - low;
|
||||
double barDrift = close - open;
|
||||
|
||||
bool highVolume = barVolume > volumeThreshold;
|
||||
bool lowDrift = barRange < InpMaxPriceDriftPoints * _Point;
|
||||
|
||||
if(highVolume && lowDrift && barDrift < 0) {
|
||||
sellAbsorptionCount++;
|
||||
}
|
||||
|
||||
if(highVolume && lowDrift && barDrift > 0) {
|
||||
buyAbsorptionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
int requiredBars = (int)MathCeil(InpAbsorptionBars / 2.0);
|
||||
|
||||
if(sellAbsorptionCount >= requiredBars && IsPriceNearPutStrike()) {
|
||||
CurrentAbsorptionState = ABSORPTION_SELL;
|
||||
@@ -679,8 +681,8 @@ bool IsAtSupport() {
|
||||
}
|
||||
|
||||
void ExecuteBuyTrade() {
|
||||
double lotSize = CalculateLotSize(ORDER_TYPE_BUY);
|
||||
if(lotSize <= 0) return;
|
||||
double lotSize = CalculateLotSize(POSITION_TYPE_BUY);
|
||||
if(lotSize <= 0) return;
|
||||
|
||||
double sl = 0, tp = 0;
|
||||
double nearestPutStrike = GetNearestPutStrike();
|
||||
@@ -715,8 +717,8 @@ void ExecuteBuyTrade() {
|
||||
}
|
||||
|
||||
void ExecuteSellTrade() {
|
||||
double lotSize = CalculateLotSize(ORDER_TYPE_SELL);
|
||||
if(lotSize <= 0) return;
|
||||
double lotSize = CalculateLotSize(POSITION_TYPE_SELL);
|
||||
if(lotSize <= 0) return;
|
||||
|
||||
double sl = 0, tp = 0;
|
||||
double nearestCallStrike = GetNearestCallStrike();
|
||||
@@ -820,8 +822,8 @@ double CalculateLotSize(ENUM_POSITION_TYPE tradeType) {
|
||||
}
|
||||
|
||||
double NormalizeLot(double lot) {
|
||||
double step = SymbolInfo.LotStep();
|
||||
return MathFloor(lot / step) * step;
|
||||
double step = SymbolInfo.LotsStep();
|
||||
return MathFloor(lot / step) * step;
|
||||
}
|
||||
|
||||
void ManagePositions() {
|
||||
@@ -1037,8 +1039,11 @@ void CreateDashboard() {
|
||||
}
|
||||
|
||||
void UpdateDashboard() {
|
||||
UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8);
|
||||
UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8);
|
||||
MqlRates rates[];
|
||||
int copied = CopyRates(_Symbol, PERIOD_M1, 0, 1, rates);
|
||||
|
||||
UpdateLabel("DB_Symbol", 20, 45, "Symbol: " + _Symbol, clrWhite, 8);
|
||||
UpdateLabel("DB_Price", 20, 65, "Price: " + DoubleToString(SpotPrice, 2), clrCyan, 8);
|
||||
|
||||
string deltaText = DoubleToString(OrderFlowDeltaPercent, 1) + "%";
|
||||
color deltaColor = OrderFlowDeltaPercent > 0 ? clrLime : (OrderFlowDeltaPercent < 0 ? clrRed : clrGray);
|
||||
@@ -1063,9 +1068,9 @@ void UpdateDashboard() {
|
||||
|
||||
UpdateLabel("DB_Absorption", 20, 105, "Absorption: " + absorptionText, absorptionColor, 8);
|
||||
|
||||
int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (int)Volume(0);
|
||||
UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) +
|
||||
" (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8);
|
||||
int currentVol = CurrentBarVolume > 0 ? CurrentBarVolume : (copied > 0 ? (int)rates[0].tick_volume : 0);
|
||||
UpdateLabel("DB_Volume", 20, 125, "Volume: " + IntegerToString(currentVol) +
|
||||
" (Avg: " + IntegerToString((int)VolumeEmaValue) + ")", clrWhite, 8);
|
||||
|
||||
string driftText = DoubleToString(PriceDrift / _Point, 1) + " pts";
|
||||
color driftColor = PriceDrift < InpMaxPriceDriftPoints * _Point ? clrLime : clrOrange;
|
||||
|
||||
@@ -1,179 +1,268 @@
|
||||
# CME OI Scraper
|
||||
|
||||
Python scraper to pull Open Interest data from CME Group QuikStrike and current gold price from investing.com.
|
||||
Python scraper that extracts Open Interest data from CME Group QuikStrike and current gold price from investing.com.
|
||||
|
||||
## What It Extracts
|
||||
|
||||
1. **OI Levels (from CME QuikStrike):**
|
||||
- Top 3 CALL strikes by OI volume
|
||||
- Top 3 PUT strikes by OI volume
|
||||
- Top 3 CALL strikes by OI volume (unique strikes)
|
||||
- Top 3 PUT strikes by OI volume (unique strikes)
|
||||
|
||||
2. **Gold Price (from investing.com):**
|
||||
- Current gold futures price (e.g., 4345.50)
|
||||
- Current gold futures price (e.g., 4476.50)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Python 3.9+
|
||||
- CME Group QuikStrike account with login credentials
|
||||
- Python 3.9 or higher
|
||||
- CME Group QuikStrike account (free registration at https://www.cmegroup.com)
|
||||
- Windows 10/11 (for batch files) or Linux/macOS
|
||||
|
||||
## Installation
|
||||
## Quick Start
|
||||
|
||||
1. Copy environment variables:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
### Windows
|
||||
|
||||
2. Edit `.env` and add your CME credentials:
|
||||
```bash
|
||||
CME_USERNAME=your_username
|
||||
CME_PASSWORD=your_password
|
||||
```
|
||||
1. **Run one-time setup:**
|
||||
```cmd
|
||||
cd C:\Path\To\oi_scraper
|
||||
setup_env.bat
|
||||
```
|
||||
|
||||
3. Install dependencies:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
playwright install chromium
|
||||
```
|
||||
2. **Run the scraper:**
|
||||
```cmd
|
||||
run_with_venv.bat
|
||||
```
|
||||
|
||||
## Usage
|
||||
### Linux/macOS
|
||||
|
||||
### Basic Scraping
|
||||
1. **Setup:**
|
||||
```bash
|
||||
cd /path/to/oi_scraper
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
playwright install chromium
|
||||
```
|
||||
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
This will:
|
||||
- Login to CME QuikStrike
|
||||
- Navigate to OI Heatmap
|
||||
- Extract top 3 CALL and PUT strikes by OI volume
|
||||
- Scrape current gold price from investing.com
|
||||
- Export to `oi_data.csv`
|
||||
|
||||
### Session Persistence
|
||||
|
||||
The scraper automatically saves your login session to `cookies.json`. This means:
|
||||
|
||||
- **First run**: Logs in with your credentials, saves cookies
|
||||
- **Subsequent runs**: Uses saved cookies if session is still valid
|
||||
- **Session expired**: Automatically logs in again and saves new cookies
|
||||
|
||||
Benefits for scheduled runs:
|
||||
- Faster execution (skips login when session is valid)
|
||||
- Reduces login attempts to CME servers
|
||||
- CME sessions typically last several days/weeks
|
||||
|
||||
To force a fresh login, delete `cookies.json`:
|
||||
```bash
|
||||
rm cookies.json
|
||||
```
|
||||
|
||||
### Output Format
|
||||
|
||||
The CSV output is compatible with the EA's `LoadOIFromCSV()` and `LoadFuturePriceFromCSV()` functions:
|
||||
|
||||
```csv
|
||||
Type,Strike,OI
|
||||
CALL,4345,155398
|
||||
CALL,4350,229137
|
||||
CALL,4360,90649
|
||||
PUT,4300,227936
|
||||
PUT,4290,270135
|
||||
PUT,4280,65839
|
||||
|
||||
[Price]
|
||||
FuturePrice,4345.50
|
||||
```
|
||||
|
||||
**Note:** The `[Price]` section contains the current gold futures price scraped from investing.com. The EA reads this value for Delta calculation.
|
||||
2. **Run:**
|
||||
```bash
|
||||
source venv/bin/activate
|
||||
python main.py
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `.env` to customize:
|
||||
### Edit `.env` File
|
||||
|
||||
- `PRODUCT_URL` - QuikStrike product page URL (requires login)
|
||||
- `CME_LOGIN_URL` - CME login page URL (default: SSO URL)
|
||||
- `TOP_N_STRIKES` - Number of top strikes to export (default: 3)
|
||||
- `HEADLESS` - Run browser in headless mode (default: false for debugging)
|
||||
- `CSV_OUTPUT_PATH` - Output CSV file path
|
||||
- `TIMEOUT_SECONDS` - Page load timeout
|
||||
Copy and edit the environment file:
|
||||
|
||||
### Available Products
|
||||
|
||||
**Gold (XAUUSD/COMEX Gold - OG|GC):**
|
||||
```
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
|
||||
```cmd
|
||||
copy .env.example .env
|
||||
notepad .env
|
||||
```
|
||||
|
||||
**Silver:**
|
||||
```
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool
|
||||
Required settings:
|
||||
```env
|
||||
CME_USERNAME=your_cme_username
|
||||
CME_PASSWORD=your_cme_password
|
||||
```
|
||||
|
||||
**SOFR (3M SOFR):**
|
||||
```
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool
|
||||
Optional settings:
|
||||
```env
|
||||
# Number of top strikes to export (default: 3)
|
||||
TOP_N_STRIKES=3
|
||||
|
||||
# Run browser without window (default: false)
|
||||
HEADLESS=false
|
||||
|
||||
# Page timeout in seconds (default: 30)
|
||||
TIMEOUT_SECONDS=30
|
||||
|
||||
# Output CSV path
|
||||
CSV_OUTPUT_PATH=./oi_data.csv
|
||||
|
||||
# Logging level: DEBUG, INFO, WARNING, ERROR
|
||||
LOG_LEVEL=INFO
|
||||
```
|
||||
|
||||
**Note:** You must be logged in to access QuikStrike data. The scraper will automatically login using credentials from `.env`.
|
||||
## Output Format
|
||||
|
||||
The scraper exports to `oi_data.csv`:
|
||||
|
||||
```csv
|
||||
Type,Strike,OI
|
||||
CALL,4375.0,147
|
||||
CALL,4450.0,173
|
||||
CALL,4500.0,176
|
||||
PUT,4435.0,49
|
||||
PUT,4400.0,102
|
||||
PUT,4515.0,150
|
||||
|
||||
[Price]
|
||||
FuturePrice,4467.8
|
||||
```
|
||||
|
||||
The `[Price]` section contains the current gold futures price scraped from investing.com.
|
||||
|
||||
## Session Persistence
|
||||
|
||||
The scraper saves login sessions to `cookies.json`:
|
||||
|
||||
- **First run:** Logs in with credentials, saves cookies
|
||||
- **Subsequent runs:** Uses saved cookies if session is valid
|
||||
- **Session expired:** Automatically re-logs in and saves new cookies
|
||||
|
||||
This makes scheduled runs faster and reduces login attempts to CME servers.
|
||||
|
||||
To force a fresh login:
|
||||
```cmd
|
||||
del cookies.json
|
||||
```
|
||||
|
||||
## Integration with EA
|
||||
|
||||
The EA reads OI data from CSV when `InpOISource = OI_SOURCE_CSV_FILE`.
|
||||
The EA reads OI data from CSV when configured:
|
||||
```mql5
|
||||
input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE;
|
||||
```
|
||||
|
||||
Place the generated `oi_data.csv` in MetaTrader's `MQL5/Files` directory.
|
||||
Copy `oi_data.csv` to your MT5 `MQL5/Files` directory:
|
||||
```
|
||||
C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\Common\MQL5\Files\oi_data.csv
|
||||
```
|
||||
|
||||
## Scheduling
|
||||
## Automatic Daily Scheduling
|
||||
|
||||
Use cron or Windows Task Scheduler to run periodically:
|
||||
### Windows Task Scheduler
|
||||
|
||||
1. **Create scheduled task:**
|
||||
- Open Task Scheduler (`taskschd.msc`)
|
||||
- Click "Create Task"
|
||||
|
||||
2. **Configure General tab:**
|
||||
- Name: `CME OI Scraper - Daily`
|
||||
- ✅ Run whether user is logged on or not
|
||||
- ✅ Run with highest privileges
|
||||
|
||||
3. **Configure Triggers tab:**
|
||||
- New → On a schedule → Daily
|
||||
- Start time: 9:00 AM (or your preferred time)
|
||||
- ✅ Enabled
|
||||
|
||||
4. **Configure Actions tab:**
|
||||
- Action: Start a program
|
||||
- Program/script:
|
||||
```
|
||||
C:\Path\To\oi_scraper\run_scheduled.bat
|
||||
```
|
||||
- Start in:
|
||||
```
|
||||
C:\Path\To\oi_scraper
|
||||
```
|
||||
|
||||
5. **Click OK to save**
|
||||
|
||||
### Linux/macOS (cron)
|
||||
|
||||
```bash
|
||||
# Run every hour
|
||||
0 * * * * cd /path/to/oi_scraper && python main.py
|
||||
# Edit crontab
|
||||
crontab -e
|
||||
|
||||
# Add line to run every day at 9 AM
|
||||
0 9 * * * cd /path/to/oi_scraper && /path/to/venv/bin/python main.py
|
||||
```
|
||||
|
||||
## Batch Files Reference
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `setup_env.bat` | One-time setup (creates virtual environment) |
|
||||
| `run_with_venv.bat` | Manual run with visible window |
|
||||
| `run_scheduled.bat` | For Task Scheduler (no window, no pause) |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Login fails:**
|
||||
### Module Not Found Errors
|
||||
|
||||
**Error:** `ModuleNotFoundError: No module named 'playwright'`
|
||||
|
||||
**Solution:**
|
||||
```cmd
|
||||
run_with_venv.bat
|
||||
```
|
||||
|
||||
The virtual environment ensures all dependencies are isolated.
|
||||
|
||||
### Login Fails
|
||||
|
||||
- Verify credentials in `.env`
|
||||
- Check if CME requires 2FA
|
||||
- Set `HEADLESS=false` to see what's happening
|
||||
- Check screenshots: `login_failed.png`, `login_error.png`, `login_success.png`
|
||||
- Check if CME requires 2FA (manual intervention needed)
|
||||
- Set `HEADLESS=false` to see browser activity
|
||||
- Check screenshots: `login_failed.png`, `login_error.png`
|
||||
|
||||
**No data extracted:**
|
||||
- Check if table structure changed
|
||||
- Increase `TIMEOUT_SECONDS`
|
||||
- Check logs for detailed errors
|
||||
- Screenshot saved as `login_debug.png` or `login_failed.png`
|
||||
### No Data Extracted
|
||||
|
||||
**Login page selectors changed:**
|
||||
- If the scraper can't find username/password inputs, CME may have updated their login page
|
||||
- Update the selectors in `login_to_cme()` function in `main.py`:
|
||||
```python
|
||||
# Example: update to match current CME login form
|
||||
page.fill('input[id="username"]', CME_USERNAME)
|
||||
page.fill('input[id="password"]', CME_PASSWORD)
|
||||
page.click('button[type="submit"]')
|
||||
```
|
||||
- Check if CME table structure changed
|
||||
- Increase `TIMEOUT_SECONDS=60` in `.env`
|
||||
- Check logs for errors
|
||||
- Screenshot saved as `login_debug.png`
|
||||
|
||||
**Browser issues:**
|
||||
- Install Chromium dependencies: `playwright install chromium`
|
||||
- Try different browser: Change `p.chromium.launch()` to `p.firefox.launch()`
|
||||
### Browser Issues
|
||||
|
||||
```cmd
|
||||
# Reinstall Chromium
|
||||
python -m playwright install chromium
|
||||
```
|
||||
|
||||
### Session Expires Frequently
|
||||
|
||||
Delete cookies to force fresh login:
|
||||
```cmd
|
||||
del cookies.json
|
||||
```
|
||||
|
||||
### Check Python Path Issues (Windows)
|
||||
|
||||
```cmd
|
||||
# Check which Python is being used
|
||||
where python
|
||||
|
||||
# Use Python launcher
|
||||
py -3 main.py
|
||||
|
||||
# Or use the virtual environment
|
||||
run_with_venv.bat
|
||||
```
|
||||
|
||||
## Finding Product IDs
|
||||
|
||||
To scrape other instruments (Silver, Crude Oil, etc.):
|
||||
|
||||
1. Visit CME QuikStrike OI Heatmap
|
||||
2. Login to your CME account
|
||||
3. Select a product from the dropdown
|
||||
4. The URL updates with the `pid` parameter
|
||||
5. Note: This scraper is configured for Gold by default
|
||||
|
||||
## Notes
|
||||
|
||||
- The scraper targets the OI Heatmap table structure
|
||||
- Only exports top N strikes by OI volume
|
||||
- Login session is not persisted (login each run)
|
||||
- Cookies could be saved for faster subsequent runs
|
||||
- Targets the OI Heatmap table structure
|
||||
- Exports top N unique strikes by OI volume
|
||||
- Uses session cookies for faster subsequent runs
|
||||
- CME sessions typically last several days to weeks
|
||||
- Virtual environment recommended to avoid Python path conflicts
|
||||
|
||||
### Finding Product IDs
|
||||
## Files
|
||||
|
||||
To find product IDs for other instruments:
|
||||
1. Visit https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html
|
||||
2. Login to your CME account
|
||||
3. Select a product from the "Products" menu
|
||||
4. The URL will update with the `pid` parameter
|
||||
5. Copy that URL to your `.env` file
|
||||
|
||||
Example: `https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html?pid=40` (Gold)
|
||||
```
|
||||
oi_scraper/
|
||||
├── main.py # Main scraper script
|
||||
├── requirements.txt # Python dependencies
|
||||
├── .env.example # Environment template
|
||||
├── .env # Your credentials (create from example)
|
||||
├── setup_env.bat # Windows: Create virtual environment
|
||||
├── run_with_venv.bat # Windows: Manual run
|
||||
├── run_scheduled.bat # Windows: Task Scheduler run
|
||||
├── oi_data.csv # Output file (generated)
|
||||
├── cookies.json # Session cookies (generated)
|
||||
└── scraper.log # Log file (generated)
|
||||
```
|
||||
|
||||
@@ -1,658 +0,0 @@
|
||||
# CME OI Scraper - Windows Setup Guide
|
||||
|
||||
Complete guide for setting up and running the CME OI scraper on Windows with automatic daily updates.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Configuration](#configuration)
|
||||
- [Manual Testing](#manual-testing)
|
||||
- [Automatic Daily Updates](#automatic-daily-updates)
|
||||
- [MetaTrader 5 Integration](#metatrader-5-integration)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Required Software
|
||||
|
||||
1. **Python 3.9 or higher**
|
||||
- Download: https://www.python.org/downloads/
|
||||
- During installation: ✅ Check "Add Python to PATH"
|
||||
|
||||
2. **CME Group QuikStrike Account**
|
||||
- Free account required: https://www.cmegroup.com/
|
||||
- Register for QuikStrike access
|
||||
- Save your username and password
|
||||
|
||||
3. **MetaTrader 5** (for EA integration)
|
||||
- Download: https://www.metatrader5.com/
|
||||
- Install on your Windows machine
|
||||
|
||||
### Verify Python Installation
|
||||
|
||||
```cmd
|
||||
python --version
|
||||
```
|
||||
|
||||
Expected output: `Python 3.9.x` or higher
|
||||
|
||||
If not found, install Python or use `py` or `python3` commands.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### Step 1: Navigate to Scraper Directory
|
||||
|
||||
Open Command Prompt (cmd) and navigate:
|
||||
|
||||
```cmd
|
||||
cd C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
|
||||
```
|
||||
|
||||
Replace `YourUsername` with your actual Windows username.
|
||||
|
||||
### Step 2: Create Environment File
|
||||
|
||||
```cmd
|
||||
copy .env.example .env
|
||||
```
|
||||
|
||||
### Step 3: Edit .env File
|
||||
|
||||
Open `.env` with Notepad:
|
||||
|
||||
```cmd
|
||||
notepad .env
|
||||
```
|
||||
|
||||
Update with your credentials:
|
||||
|
||||
```env
|
||||
# CME Group QuikStrike Login Credentials
|
||||
CME_USERNAME=your_actual_username_here
|
||||
CME_PASSWORD=your_actual_password_here
|
||||
|
||||
# Product Configuration (Gold)
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
|
||||
|
||||
# Output Settings
|
||||
CSV_OUTPUT_PATH=./oi_data.csv
|
||||
TOP_N_STRIKES=3
|
||||
|
||||
# Scraping Settings
|
||||
HEADLESS=false
|
||||
TIMEOUT_SECONDS=30
|
||||
RETRY_ATTEMPTS=3
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=INFO
|
||||
```
|
||||
|
||||
**Save and close** (Ctrl+S, then Alt+F4).
|
||||
|
||||
### Step 4: Install Python Dependencies
|
||||
|
||||
```cmd
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
Expected output: Successfully installed playwright, python-dotenv, pandas
|
||||
|
||||
### Step 5: Install Playwright Browser
|
||||
|
||||
```cmd
|
||||
playwright install chromium
|
||||
```
|
||||
|
||||
Expected output: Downloading Chromium... [progress bar]
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### Available Products
|
||||
|
||||
**Gold (XAUUSD/COMEX Gold - OG|GC):**
|
||||
```env
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool
|
||||
```
|
||||
|
||||
**Silver:**
|
||||
```env
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool
|
||||
```
|
||||
|
||||
**SOFR (3M SOFR):**
|
||||
```env
|
||||
PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool
|
||||
```
|
||||
|
||||
### Configuration Options
|
||||
|
||||
| Setting | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `TOP_N_STRIKES` | Number of top strikes to export | 3 |
|
||||
| `HEADLESS` | Run browser without window (true/false) | false |
|
||||
| `TIMEOUT_SECONDS` | Page load timeout in seconds | 30 |
|
||||
| `CSV_OUTPUT_PATH` | Output CSV file path | ./oi_data.csv |
|
||||
| `LOG_LEVEL` | DEBUG, INFO, WARNING, ERROR | INFO |
|
||||
|
||||
---
|
||||
|
||||
## Manual Testing
|
||||
|
||||
### Run Scraper Manually
|
||||
|
||||
```cmd
|
||||
python main.py
|
||||
```
|
||||
|
||||
Expected output:
|
||||
```
|
||||
INFO:__main__:Cookies loaded from file
|
||||
INFO:__main__:Using existing session (cookies)
|
||||
INFO:__main__:Navigating to OI Heatmap: https://...
|
||||
INFO:__main__:Extracting OI data from Gold matrix table...
|
||||
INFO:__main__:Extracted 6 OI levels
|
||||
INFO:__main__:Exported OI data to ./oi_data.csv
|
||||
```
|
||||
|
||||
### Check Output
|
||||
|
||||
**1. Verify CSV created:**
|
||||
|
||||
```cmd
|
||||
dir oi_data.csv
|
||||
```
|
||||
|
||||
**2. View CSV content:**
|
||||
|
||||
```cmd
|
||||
notepad oi_data.csv
|
||||
```
|
||||
|
||||
Expected format:
|
||||
```csv
|
||||
Type,Strike,OI
|
||||
CALL,4400,6193
|
||||
CALL,4300,3826
|
||||
CALL,4350,1983
|
||||
PUT,4400,5559
|
||||
PUT,4300,2988
|
||||
PUT,4350,1214
|
||||
```
|
||||
|
||||
### Check Logs
|
||||
|
||||
```cmd
|
||||
type scraper.log
|
||||
```
|
||||
|
||||
Or view in Notepad:
|
||||
|
||||
```cmd
|
||||
notepad scraper.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Automatic Daily Updates
|
||||
|
||||
### Option 1: Windows Task Scheduler (Recommended)
|
||||
|
||||
#### Step 1: Create Batch File Wrapper
|
||||
|
||||
Create `run_scraper.bat` in the oi_scraper directory:
|
||||
|
||||
```cmd
|
||||
@echo off
|
||||
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
|
||||
|
||||
echo Starting CME OI Scraper at %date% %time% >> scraper.log
|
||||
echo ---------------------------------------- >> scraper.log
|
||||
|
||||
python main.py >> scraper.log 2>&1
|
||||
|
||||
if %ERRORLEVEL% EQU 0 (
|
||||
echo %date% %time%: Scraper completed successfully >> scraper.log
|
||||
) else (
|
||||
echo %date% %time%: Scraper failed with error %ERRORLEVEL% >> scraper.log
|
||||
)
|
||||
|
||||
echo ---------------------------------------- >> scraper.log
|
||||
```
|
||||
|
||||
Replace `YourUsername` with your actual username.
|
||||
|
||||
#### Step 2: Open Task Scheduler
|
||||
|
||||
Press `Win + R`, type `taskschd.msc`, press Enter
|
||||
|
||||
Or: Start → Windows Administrative Tools → Task Scheduler
|
||||
|
||||
#### Step 3: Create Task
|
||||
|
||||
1. Click **"Create Basic Task"** on right sidebar
|
||||
2. **Name:** `CME OI Scraper - Daily`
|
||||
3. **Description:** `Update OI data from CME QuikStrike every day at 9 AM`
|
||||
4. Click **Next**
|
||||
|
||||
#### Step 4: Set Trigger
|
||||
|
||||
1. **Trigger:** Select "Daily"
|
||||
2. **Start date:** Today's date
|
||||
3. **Start time:** 9:00:00 AM (or your preferred time)
|
||||
4. Click **Next**
|
||||
|
||||
#### Step 5: Set Action
|
||||
|
||||
1. **Action:** Select "Start a program"
|
||||
2. **Program/script:**
|
||||
```
|
||||
C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.bat
|
||||
```
|
||||
3. **Start in (optional):**
|
||||
```
|
||||
C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
|
||||
```
|
||||
4. Click **Next**
|
||||
|
||||
#### Step 6: Finish
|
||||
|
||||
1. Review settings
|
||||
2. Check "Open the Properties dialog for this task when I click Finish"
|
||||
3. Click **Finish**
|
||||
|
||||
#### Step 7: Configure Advanced Settings (Optional)
|
||||
|
||||
In the Properties dialog:
|
||||
|
||||
- **General tab:**
|
||||
- ✅ Run whether user is logged on or not
|
||||
- ✅ Do not store password (if using Windows authentication)
|
||||
- ✅ Run with highest privileges
|
||||
|
||||
- **Conditions tab:**
|
||||
- ✅ Start the task only if the computer is on AC power
|
||||
- ✅ Stop if the computer switches to battery power
|
||||
- ✅ Wake the computer to run this task
|
||||
|
||||
- **Settings tab:**
|
||||
- ✅ Allow task to be run on demand
|
||||
- ❌ Stop the task if it runs longer than: 30 minutes
|
||||
- ✅ If the task fails, restart every: 5 minutes (up to 3 times)
|
||||
|
||||
Click **OK** to save settings.
|
||||
|
||||
#### Step 8: Test Task
|
||||
|
||||
1. In Task Scheduler, find "CME OI Scraper - Daily"
|
||||
2. Right-click → **Run**
|
||||
3. Check `scraper.log` after a minute:
|
||||
```cmd
|
||||
type scraper.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Option 2: PowerShell Script (Advanced)
|
||||
|
||||
#### Step 1: Create PowerShell Script
|
||||
|
||||
Save as `run_scraper.ps1`:
|
||||
|
||||
```powershell
|
||||
# Script configuration
|
||||
$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper"
|
||||
$logFile = "$scriptPath\scraper.log"
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
|
||||
|
||||
# Navigate to script directory
|
||||
cd $scriptPath
|
||||
|
||||
try {
|
||||
# Run Python scraper
|
||||
Write-Output "$timestamp: Starting CME OI Scraper" | Add-Content $logFile
|
||||
& python main.py *>> $logFile 2>&1
|
||||
|
||||
# Check if CSV was created
|
||||
if (Test-Path "oi_data.csv") {
|
||||
$fileInfo = Get-Item "oi_data.csv"
|
||||
Write-Output "$timestamp: Scraper completed successfully (CSV updated: $($fileInfo.LastWriteTime))" | Add-Content $logFile
|
||||
} else {
|
||||
Write-Output "$timestamp: WARNING - CSV file not created" | Add-Content $logFile
|
||||
}
|
||||
} catch {
|
||||
$errorMsg = $_.Exception.Message
|
||||
Write-Output "$timestamp: ERROR - $errorMsg" | Add-Content $logFile
|
||||
exit 1
|
||||
}
|
||||
```
|
||||
|
||||
#### Step 2: Update Task Scheduler to Use PowerShell
|
||||
|
||||
Same steps as Option 1, but:
|
||||
|
||||
- **Program/script:** `powershell.exe`
|
||||
- **Add arguments:**
|
||||
```
|
||||
-ExecutionPolicy Bypass -File "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.ps1"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## MetaTrader 5 Integration
|
||||
|
||||
### Find MT5 Files Directory
|
||||
|
||||
MT5 data directory location:
|
||||
|
||||
```
|
||||
C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Terminal_ID]\MQL5\Files\
|
||||
```
|
||||
|
||||
**To find your Terminal_ID:**
|
||||
|
||||
1. Open MT5
|
||||
2. Click **File** → **Open Data Folder**
|
||||
3. Navigate to `Terminal\[Your_Terminal_ID]\MQL5\Files\`
|
||||
|
||||
### Update Batch File to Copy to MT5
|
||||
|
||||
Edit `run_scraper.bat`:
|
||||
|
||||
```cmd
|
||||
@echo off
|
||||
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
|
||||
|
||||
echo Starting CME OI Scraper at %date% %time% >> scraper.log
|
||||
echo ---------------------------------------- >> scraper.log
|
||||
|
||||
python main.py >> scraper.log 2>&1
|
||||
|
||||
if %ERRORLEVEL% EQU 0 (
|
||||
if exist oi_data.csv (
|
||||
echo Copying OI data to MT5... >> scraper.log
|
||||
copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
|
||||
|
||||
if %ERRORLEVEL% EQU 0 (
|
||||
echo %date% %time%: Scraper completed - OI data copied to MT5 >> scraper.log
|
||||
) else (
|
||||
echo %date% %time%: ERROR - Failed to copy to MT5 >> scraper.log
|
||||
)
|
||||
) else (
|
||||
echo %date% %time%: ERROR - oi_data.csv not found >> scraper.log
|
||||
)
|
||||
) else (
|
||||
echo %date% %time%: ERROR - Scraper failed with error %ERRORLEVEL% >> scraper.log
|
||||
)
|
||||
|
||||
echo ---------------------------------------- >> scraper.log
|
||||
```
|
||||
|
||||
Replace `[Your_Terminal_ID]` with your actual MT5 terminal ID.
|
||||
|
||||
### Update EA Configuration
|
||||
|
||||
In your EA (`OI_MeanReversion_Pro_XAUUSD_A.mq5`), set:
|
||||
|
||||
```mql5
|
||||
input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE; // Load from CSV file
|
||||
```
|
||||
|
||||
The EA will automatically read `oi_data.csv` from its Files directory.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Python Not Found
|
||||
|
||||
**Error:** `'python' is not recognized as an internal or external command`
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. Use full path to Python:
|
||||
```cmd
|
||||
C:\Users\YourUsername\AppData\Local\Programs\Python\Python312\python.exe main.py
|
||||
```
|
||||
|
||||
2. Use `py` launcher:
|
||||
```cmd
|
||||
py main.py
|
||||
```
|
||||
|
||||
3. Reinstall Python with "Add to PATH" option
|
||||
|
||||
### Module Import Errors
|
||||
|
||||
**Error:** `ModuleNotFoundError: No module named 'playwright'`
|
||||
|
||||
**Solution:**
|
||||
```cmd
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### Login Fails
|
||||
|
||||
**Error:** `Login failed - still on login page`
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. Check credentials in `.env` file:
|
||||
```cmd
|
||||
notepad .env
|
||||
```
|
||||
|
||||
2. Check login screenshots:
|
||||
- `login_failed.png` - Shows login page
|
||||
- `login_error.png` - Shows error during login
|
||||
- `login_success.png` - Confirms successful login
|
||||
|
||||
3. Manually test login at: https://www.cmegroup.com/
|
||||
|
||||
4. Check if 2FA is required (CME may require additional authentication)
|
||||
|
||||
### No Data Extracted
|
||||
|
||||
**Warning:** `No CALL OI data extracted` or `No PUT OI data extracted`
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. Check if you're logged in:
|
||||
- Delete `cookies.json` to force fresh login
|
||||
- Run scraper manually with `HEADLESS=false` in `.env`
|
||||
|
||||
2. Check if page structure changed:
|
||||
- View screenshots to see actual page content
|
||||
- Check if Gold product URL is correct
|
||||
|
||||
3. Increase timeout:
|
||||
```env
|
||||
TIMEOUT_SECONDS=60
|
||||
```
|
||||
|
||||
### Task Not Running
|
||||
|
||||
**Issue:** Task Scheduler doesn't execute the task
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. Check task history:
|
||||
- Task Scheduler → Right-click task → Properties → History tab
|
||||
- Look for errors in the log
|
||||
|
||||
2. Test manually:
|
||||
- Right-click task → Run
|
||||
- Check `scraper.log` for output
|
||||
|
||||
3. Check account permissions:
|
||||
- Ensure task is set to run with your Windows account
|
||||
- Check "Run whether user is logged on or not"
|
||||
|
||||
4. Check Windows Event Viewer:
|
||||
- Event Viewer → Windows Logs → Application
|
||||
- Look for Task Scheduler errors
|
||||
|
||||
### Session Expiration
|
||||
|
||||
**Issue:** Session expires after some time
|
||||
|
||||
**Solution:**
|
||||
The scraper will automatically re-login when cookies expire. No manual action needed.
|
||||
|
||||
To force fresh login:
|
||||
```cmd
|
||||
del cookies.json
|
||||
```
|
||||
|
||||
### Check Logs
|
||||
|
||||
**View recent logs:**
|
||||
|
||||
```cmd
|
||||
type scraper.log | more
|
||||
```
|
||||
|
||||
**View last 20 lines:**
|
||||
|
||||
```cmd
|
||||
powershell "Get-Content scraper.log -Tail 20"
|
||||
```
|
||||
|
||||
**Search for errors:**
|
||||
|
||||
```cmd
|
||||
findstr /C:"ERROR" scraper.log
|
||||
```
|
||||
|
||||
### Verify CSV Output
|
||||
|
||||
**Check if CSV is valid:**
|
||||
|
||||
```cmd
|
||||
python -c "import pandas as pd; print(pd.read_csv('oi_data.csv'))"
|
||||
```
|
||||
|
||||
**Check file size:**
|
||||
|
||||
```cmd
|
||||
dir oi_data.csv
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Advanced Options
|
||||
|
||||
### Run Multiple Times Per Day
|
||||
|
||||
**Edit Task Scheduler Trigger:**
|
||||
|
||||
1. Open task properties → Triggers tab
|
||||
2. Edit existing trigger → Click "New" to add additional
|
||||
3. Set different times:
|
||||
- 9:00 AM
|
||||
- 12:00 PM
|
||||
- 3:00 PM
|
||||
- 6:00 PM
|
||||
|
||||
### Run on Market Days Only
|
||||
|
||||
**Create separate batch file:**
|
||||
|
||||
```cmd
|
||||
@echo off
|
||||
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
|
||||
|
||||
REM Check if today is weekday (1=Monday, 5=Friday)
|
||||
for /f "skip=1 tokens=*" %%a in ('wmic path win32_localtime get dayofweek /value') do set DAY=%%a
|
||||
|
||||
if %DAY% LSS 1 goto END
|
||||
if %DAY% GTR 5 goto END
|
||||
|
||||
REM Run scraper
|
||||
python main.py >> scraper.log 2>&1
|
||||
|
||||
:END
|
||||
```
|
||||
|
||||
### Email Notifications
|
||||
|
||||
**Use PowerShell to send email on completion:**
|
||||
|
||||
```powershell
|
||||
# Add to run_scraper.ps1 at the end
|
||||
$smtpServer = "smtp.gmail.com"
|
||||
$smtpPort = 587
|
||||
$smtpUser = "your_email@gmail.com"
|
||||
$smtpPass = "your_password"
|
||||
$from = "CME OI Scraper <your_email@gmail.com>"
|
||||
$to = "your_email@gmail.com"
|
||||
$subject = "CME OI Scraper - %date%"
|
||||
|
||||
if ($errorOccurred) {
|
||||
$body = "CME OI Scraper failed. Check logs for details."
|
||||
} else {
|
||||
$body = "CME OI Scraper completed successfully.`n`nUpdated files:`n- oi_data.csv"
|
||||
}
|
||||
|
||||
$message = New-Object System.Net.Mail.MailMessage $from, $to
|
||||
$message.Subject = $subject
|
||||
$message.Body = $body
|
||||
|
||||
$smtp = New-Object System.Net.Mail.SmtpClient $smtpServer, $smtpPort
|
||||
$smtp.EnableSsl = $true
|
||||
$smtp.Credentials = New-Object System.Net.NetworkCredential $smtpUser, $smtpPass
|
||||
|
||||
$smtp.Send($message)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
**Quick Start Checklist:**
|
||||
|
||||
- [ ] Python 3.9+ installed
|
||||
- [ ] CME QuikStrike account created
|
||||
- [ ] `.env` file configured with credentials
|
||||
- [ ] Dependencies installed (`pip install -r requirements.txt`)
|
||||
- [ ] Playwright browser installed (`playwright install chromium`)
|
||||
- [ ] Manual test successful (`python main.py`)
|
||||
- [ ] `oi_data.csv` created and valid
|
||||
- [ ] Task Scheduler task created
|
||||
- [ ] Task tested manually
|
||||
- [ ] CSV copied to MT5 Files directory
|
||||
- [ ] EA configured to use CSV file
|
||||
|
||||
**Daily Workflow:**
|
||||
|
||||
1. Task Scheduler runs at 9:00 AM
|
||||
2. Batch file executes Python scraper
|
||||
3. Scraper logs in with saved cookies (or fresh login)
|
||||
4. OI data extracted and saved to `oi_data.csv`
|
||||
5. CSV copied to MT5 Files directory
|
||||
6. EA reads updated OI data
|
||||
7. EA uses new OI levels for trading
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
For issues or questions:
|
||||
|
||||
1. Check `scraper.log` for detailed error messages
|
||||
2. Review screenshots (login_failed.png, login_error.png)
|
||||
3. Verify `.env` configuration
|
||||
4. Test manually without Task Scheduler
|
||||
5. Check Windows Event Viewer for system errors
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** January 4, 2026
|
||||
**Version:** 1.0
|
||||
**Platform:** Windows 10/11
|
||||
@@ -1,3 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
CME OI Scraper - Extracts Open Interest data from CME QuikStrike and gold price from investing.com
|
||||
Usage: python main.py
|
||||
Requires: pip install -r requirements.txt
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
@@ -212,19 +220,24 @@ def extract_oi_data(page):
|
||||
|
||||
if call_levels:
|
||||
call_df = pd.DataFrame(call_levels)
|
||||
call_df = call_df.groupby("Strike", as_index=False).agg({"OI": "max"})
|
||||
call_df = call_df.nlargest(TOP_N_STRIKES, "OI")
|
||||
call_df = call_df.drop_duplicates(subset="Strike", keep="first")
|
||||
call_df = call_df.sort_values("OI")
|
||||
call_df = call_df.tail(TOP_N_STRIKES)
|
||||
call_df["Type"] = "CALL"
|
||||
else:
|
||||
call_df = pd.DataFrame()
|
||||
|
||||
if put_levels:
|
||||
put_df = pd.DataFrame(put_levels)
|
||||
put_df = put_df.groupby("Strike", as_index=False).agg({"OI": "max"})
|
||||
put_df = put_df.nlargest(TOP_N_STRIKES, "OI")
|
||||
put_df = put_df.drop_duplicates(subset="Strike", keep="first")
|
||||
put_df = put_df.sort_values("OI")
|
||||
put_df = put_df.tail(TOP_N_STRIKES)
|
||||
put_df["Type"] = "PUT"
|
||||
else:
|
||||
put_df = pd.DataFrame()
|
||||
|
||||
result_df = pd.concat([call_df, put_df], ignore_index=True)
|
||||
result_df = pd.concat([call_df, put_df])
|
||||
result_df = result_df[["Type", "Strike", "OI"]]
|
||||
|
||||
logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted")
|
||||
return result_df
|
||||
@@ -327,7 +340,7 @@ def run_scraper():
|
||||
oi_data = extract_oi_data(page)
|
||||
save_cookies(context)
|
||||
|
||||
if not oi_data.empty:
|
||||
if len(oi_data) > 0:
|
||||
logger.info("Extracting gold price from investing.com...")
|
||||
future_price = scrape_investing_gold_price(page)
|
||||
logger.info(f"Gold price extracted: {future_price}")
|
||||
|
||||
13
oi_scraper/run_scheduled.bat
Normal file
13
oi_scraper/run_scheduled.bat
Normal file
@@ -0,0 +1,13 @@
|
||||
@echo off
|
||||
REM ==========================================
|
||||
REM CME OI Scraper - Scheduled Task Version
|
||||
REM For use with Windows Task Scheduler
|
||||
REM ==========================================
|
||||
|
||||
REM Navigate to script directory
|
||||
cd /d %~dp0
|
||||
|
||||
REM Activate virtual environment and run scraper (no pause)
|
||||
call venv\Scripts\activate.bat
|
||||
python main.py
|
||||
exit %ERRORLEVEL%
|
||||
@@ -1,47 +1,21 @@
|
||||
@echo off
|
||||
REM ==========================================
|
||||
REM CME OI Scraper - Automatic Daily Runner
|
||||
REM CME OI Scraper - Run with Virtual Environment
|
||||
REM ==========================================
|
||||
|
||||
REM Change to script directory
|
||||
cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper
|
||||
REM Navigate to script directory
|
||||
cd /d %~dp0
|
||||
|
||||
echo ==========================================
|
||||
echo CME OI Scraper - Daily Update
|
||||
echo CME OI Scraper
|
||||
echo ==========================================
|
||||
echo Started at: %date% %time%
|
||||
echo ========================================== >> scraper.log
|
||||
|
||||
REM Activate virtual environment
|
||||
call venv\Scripts\activate.bat
|
||||
|
||||
REM Run Python scraper
|
||||
python main.py >> scraper.log 2>&1
|
||||
python main.py
|
||||
|
||||
REM Check if scraper succeeded
|
||||
if %ERRORLEVEL% EQU 0 (
|
||||
echo [%date% %time%] Scraper completed successfully >> scraper.log
|
||||
|
||||
REM Check if CSV file was created
|
||||
if exist oi_data.csv (
|
||||
echo [%date% %time%] CSV file created successfully >> scraper.log
|
||||
|
||||
REM Copy to MetaTrader 5 Files directory
|
||||
REM Update this path to your actual MT5 directory
|
||||
copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv"
|
||||
|
||||
if %ERRORLEVEL% EQU 0 (
|
||||
echo [%date% %time%] CSV copied to MT5 Files directory >> scraper.log
|
||||
) else (
|
||||
echo [%date% %time%] ERROR: Failed to copy CSV to MT5 directory >> scraper.log
|
||||
)
|
||||
) else (
|
||||
echo [%date% %time%] WARNING: oi_data.csv not found >> scraper.log
|
||||
)
|
||||
) else (
|
||||
echo [%date% %time%] ERROR: Scraper failed with error code %ERRORLEVEL% >> scraper.log
|
||||
)
|
||||
|
||||
echo ==========================================
|
||||
echo Completed at: %date% %time%
|
||||
echo ==========================================
|
||||
|
||||
REM Keep window open for 5 seconds to see any errors
|
||||
REM Pause for 5 seconds if running manually (not scheduled)
|
||||
if "%1"=="--scheduled" goto :eof
|
||||
timeout /t 5
|
||||
|
||||
21
oi_scraper/run_with_venv.bat
Normal file
21
oi_scraper/run_with_venv.bat
Normal file
@@ -0,0 +1,21 @@
|
||||
@echo off
|
||||
REM ==========================================
|
||||
REM CME OI Scraper - Manual Run with Virtual Environment
|
||||
REM ==========================================
|
||||
|
||||
REM Navigate to script directory
|
||||
cd /d %~dp0
|
||||
|
||||
echo ==========================================
|
||||
echo CME OI Scraper - Manual Run
|
||||
echo ==========================================
|
||||
|
||||
REM Activate virtual environment
|
||||
call venv\Scripts\activate.bat
|
||||
|
||||
REM Run Python scraper
|
||||
python main.py
|
||||
|
||||
echo.
|
||||
echo Scraper completed. Check oi_data.csv for results.
|
||||
timeout /t 5
|
||||
34
oi_scraper/setup_env.bat
Normal file
34
oi_scraper/setup_env.bat
Normal file
@@ -0,0 +1,34 @@
|
||||
@echo off
|
||||
REM ==========================================
|
||||
REM CME OI Scraper - Virtual Environment Setup
|
||||
REM ==========================================
|
||||
|
||||
echo ==========================================
|
||||
echo Setting up Python Virtual Environment
|
||||
echo ==========================================
|
||||
|
||||
REM Navigate to script directory
|
||||
cd /d %~dp0
|
||||
|
||||
REM Create virtual environment
|
||||
echo Creating virtual environment...
|
||||
py -3 -m venv venv
|
||||
|
||||
REM Activate virtual environment and install dependencies
|
||||
echo Installing dependencies...
|
||||
call venv\Scripts\activate.bat
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
REM Install playwright browser
|
||||
echo Installing Playwright browser...
|
||||
python -m playwright install chromium
|
||||
|
||||
echo ==========================================
|
||||
echo Setup Complete!
|
||||
echo ==========================================
|
||||
echo.
|
||||
echo To run the scraper, use: run_with_venv.bat
|
||||
echo.
|
||||
|
||||
pause
|
||||
Reference in New Issue
Block a user