refactor(oi): improve data extraction and consolidate documentation

- Fix MQL5 API usage in EA to use correct CopyRates and POSITION_TYPE enums
- Refactor scraper data extraction to use drop_duplicates for unique strikes
- Consolidate Windows setup guide into main README
- Add virtual environment batch files for easier setup and execution
- Simplify run_scraper.bat to focus on core execution
- Normalize lot calculation to use SymbolInfo.LotsStep()
This commit is contained in:
Kunthawat Greethong
2026-01-06 20:18:12 +07:00
parent 2e8e07ed17
commit b7c0e68fa8
8 changed files with 386 additions and 895 deletions

View File

@@ -1,3 +1,11 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
CME OI Scraper - Extracts Open Interest data from CME QuikStrike and gold price from investing.com
Usage: python main.py
Requires: pip install -r requirements.txt
"""
import os
import logging
import json
@@ -212,19 +220,24 @@ def extract_oi_data(page):
if call_levels:
call_df = pd.DataFrame(call_levels)
call_df = call_df.groupby("Strike", as_index=False).agg({"OI": "max"})
call_df = call_df.nlargest(TOP_N_STRIKES, "OI")
call_df = call_df.drop_duplicates(subset="Strike", keep="first")
call_df = call_df.sort_values("OI")
call_df = call_df.tail(TOP_N_STRIKES)
call_df["Type"] = "CALL"
else:
call_df = pd.DataFrame()
if put_levels:
put_df = pd.DataFrame(put_levels)
put_df = put_df.groupby("Strike", as_index=False).agg({"OI": "max"})
put_df = put_df.nlargest(TOP_N_STRIKES, "OI")
put_df = put_df.drop_duplicates(subset="Strike", keep="first")
put_df = put_df.sort_values("OI")
put_df = put_df.tail(TOP_N_STRIKES)
put_df["Type"] = "PUT"
else:
put_df = pd.DataFrame()
result_df = pd.concat([call_df, put_df], ignore_index=True)
result_df = pd.concat([call_df, put_df])
result_df = result_df[["Type", "Strike", "OI"]]
logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted")
return result_df
@@ -327,7 +340,7 @@ def run_scraper():
oi_data = extract_oi_data(page)
save_cookies(context)
if not oi_data.empty:
if len(oi_data) > 0:
logger.info("Extracting gold price from investing.com...")
future_price = scrape_investing_gold_price(page)
logger.info(f"Gold price extracted: {future_price}")