refactor(oi): improve data extraction and consolidate documentation
- Fix MQL5 API usage in EA to use correct CopyRates and POSITION_TYPE enums - Refactor scraper data extraction to use drop_duplicates for unique strikes - Consolidate Windows setup guide into main README - Add virtual environment batch files for easier setup and execution - Simplify run_scraper.bat to focus on core execution - Normalize lot calculation to use SymbolInfo.LotsStep()
This commit is contained in:
@@ -1,3 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
CME OI Scraper - Extracts Open Interest data from CME QuikStrike and gold price from investing.com
|
||||
Usage: python main.py
|
||||
Requires: pip install -r requirements.txt
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
@@ -212,19 +220,24 @@ def extract_oi_data(page):
|
||||
|
||||
if call_levels:
|
||||
call_df = pd.DataFrame(call_levels)
|
||||
call_df = call_df.groupby("Strike", as_index=False).agg({"OI": "max"})
|
||||
call_df = call_df.nlargest(TOP_N_STRIKES, "OI")
|
||||
call_df = call_df.drop_duplicates(subset="Strike", keep="first")
|
||||
call_df = call_df.sort_values("OI")
|
||||
call_df = call_df.tail(TOP_N_STRIKES)
|
||||
call_df["Type"] = "CALL"
|
||||
else:
|
||||
call_df = pd.DataFrame()
|
||||
|
||||
if put_levels:
|
||||
put_df = pd.DataFrame(put_levels)
|
||||
put_df = put_df.groupby("Strike", as_index=False).agg({"OI": "max"})
|
||||
put_df = put_df.nlargest(TOP_N_STRIKES, "OI")
|
||||
put_df = put_df.drop_duplicates(subset="Strike", keep="first")
|
||||
put_df = put_df.sort_values("OI")
|
||||
put_df = put_df.tail(TOP_N_STRIKES)
|
||||
put_df["Type"] = "PUT"
|
||||
else:
|
||||
put_df = pd.DataFrame()
|
||||
|
||||
result_df = pd.concat([call_df, put_df], ignore_index=True)
|
||||
result_df = pd.concat([call_df, put_df])
|
||||
result_df = result_df[["Type", "Strike", "OI"]]
|
||||
|
||||
logger.info(f"Final top {TOP_N_STRIKES} unique strikes for CALL and PUT extracted")
|
||||
return result_df
|
||||
@@ -327,7 +340,7 @@ def run_scraper():
|
||||
oi_data = extract_oi_data(page)
|
||||
save_cookies(context)
|
||||
|
||||
if not oi_data.empty:
|
||||
if len(oi_data) > 0:
|
||||
logger.info("Extracting gold price from investing.com...")
|
||||
future_price = scrape_investing_gold_price(page)
|
||||
logger.info(f"Gold price extracted: {future_price}")
|
||||
|
||||
Reference in New Issue
Block a user