From 28a4546cd892475986def252753a5297e2851fea Mon Sep 17 00:00:00 2001 From: Kunthawat Greethong Date: Sun, 4 Jan 2026 17:35:14 +0700 Subject: [PATCH] feat(oi): add open interest scraper module add new oi_scraper directory for collecting open interest data and update the main EA to integrate with the scraper functionality --- OI_MeanReversion_Pro_XAUUSD_A.mq5 | Bin 160974 -> 163932 bytes oi_scraper/.env.example | 27 ++ oi_scraper/.gitignore | 31 ++ oi_scraper/README.md | 178 ++++++++ oi_scraper/WINDOWS_SETUP.md | 658 ++++++++++++++++++++++++++++++ oi_scraper/main.py | 257 ++++++++++++ oi_scraper/requirements.txt | 3 + oi_scraper/run_scraper.bat | 47 +++ oi_scraper/run_scraper.ps1 | 77 ++++ 9 files changed, 1278 insertions(+) create mode 100644 oi_scraper/.env.example create mode 100644 oi_scraper/.gitignore create mode 100644 oi_scraper/README.md create mode 100644 oi_scraper/WINDOWS_SETUP.md create mode 100644 oi_scraper/main.py create mode 100644 oi_scraper/requirements.txt create mode 100644 oi_scraper/run_scraper.bat create mode 100644 oi_scraper/run_scraper.ps1 diff --git a/OI_MeanReversion_Pro_XAUUSD_A.mq5 b/OI_MeanReversion_Pro_XAUUSD_A.mq5 index 979b95f03cd30540b4596afda1183a3d04c11759..66a7a9b0aea47d74f8ae257c681c4fad3a7ad151 100644 GIT binary patch delta 23821 zcmcgU33ye-)o+FnLJ~qq^0L3=y@ZgB?1UviNZ7-cg|H(eEP+4>2_T|GNL*Tq8)1~A zKs6MqRs}zZ@Zs)i-WA7@%%(AIM@U7m7e*Wo@e>3@);r)^$1l<+k4p6(up2TN-1Nl zWu0Y>rP8t(|9Y$c(ky9~6#36?DX>&yTb1Qz%M!~{%UX-KsO=dm&Upmkcq;kzB$!kx z=@}(U;2D`-$-&gWzS^1}4v3;ob?UvyR&cerQIzn0KH5s4z(Q)F8S2~th?-YEd^`^G8 z-Zs(HI~7Z(doS#*j~1(f!$gsH3D)oRZp3G~Emnk|jSlbDC7@%7cK`s<}2ajK8M`j*}&MlS`3raUjPt&f+XrYON*t`8A{+F?9UeBCD) zn^~Yu4D?&Bc0BC2UM~eeXbTGx8~T=L1nf&Br$y)Z6QOAVTm18|bjcbo+WhCMbv6A6 z{g!^qDwGt*rtOKxg$$AeyR|Vfmy@M;|bBNd;G6+j8{o>%_-7IK(C^0bMTpi&FI~!RL zOnZ!ZihDzc%3kr}a_Ax~(cD#Gv-OgnXbT%Fdqj)U@D&Cu&WF=Z5Ed~J#BK^~jS5#H zmZ&glA}f^f@bDvSfQYmaxN;jwPrFU%ojjrbA?6MCUCdV;i^?-h6dFz7^P`LLiOUg( zqf2RQutTKUY3Hy6*Eo+6d4XQyL%X(~(T;WMfC~=7B|qkF73f77@K?_BZ_-k zMQUuQ@&>i39Hlun9J|~eI}D%C$EK(qefm3ucmMu67@H{SAAmg;_s_y7^iRX*8~uml zvqxMuK8M9+sC`$*CF1*MnPD(+Df1J@<2R_HF z%W{({c3OK8s)u!Mtpov8334?H6x(t=Md^US2>kSbXmL3=Qq=o+$+t1`nI)P>X`*p@ zK$a>%(Fqx1!6>iZ>6TPWI!!95#wkS63Hd%WjXITR$qx~CB#<83lR)d-lMn|*%b5|t zo`eY4x`g=}U{Y1?G8Q0CB@jyZD(+7VR|&K~kyOZ^5{Zy~lZcR`k_dF)qzJJsiA42@ zq#^kHAc?l*gCrX)cklr4@6SDP3QaxDN{+|Y&B@f~^JJSiFvib-z~{*sre(BGwTj$< zq(KS?=Hs(*-~@cWC!da#0&VM3BCz(Jl&RE~LYr$%rQWkrX_m%Rs(Cv#TPn)Mmesg_ zt1WAwd?ZKBlBO_~wz%)nNb86qzJ z-e2SfdWy14FLnGX+4fi_>GaPsN8vL$i=Hdx^XV+Y`RS}!(e#$DXnW05v}V!vVGa1_ zA}?Re)DT&n9w+V}L}u`bLBwvK4WdbFhXjiBY!VYTEJCzC?`1`Mga6Rd;6J@2RwHtC zmA=9++YkHh&n6jtPd+mS$BCxbNMbSuQ!TwOI8X0?9vmwUoK;J`GA%I{vSoiBOv2V{ zNSxelbsjNf2<=zx5aP4P&5+~~eMbc2hvK{d)8tRRuc|&RE9~%-a&K&XT zsldiJpV26Xx2kg~Yw{FVbzg-Q(qAI^T+s>OyQ8mVkW@4y(myY)-to35q+> zJ}FFmokK=QdH-Q}gl?`KBMPv7<%os&d`mu)N6s;=&@m!e92rTna&{zf<=K(Z;^O5f zw?6f`rMj)(lf&ZC273yxQAt{*aKy)HcGp839u+R?M|l_;Q?M>de`kM8+}_7Sc<1R= z&@h$7qC)W#WqH9`Kc%uMufMS(o<=^I7uq*V6=7|&Xekr(aXa9FJeK9U0@zy4&@-Yo z8?@%JP}dG(>}YCt>}OU{JvvG5gaR5JA2_&$MZ?9JNTE}s2Vu9yJD5$_$ApIH%!nS3 zvy6TpVsqOV(sz%HAvN&VF_Yb9Tt1eVr*OPgY#VDgPbgj*OGbI#xCl5ffCu*=1v_tC zZ5Pv7#n9amlwvV8)%6O$(-oA7IxfVu%%9zX+eZbA-r=r?h) z3yPH!vpa(eXT4d7I5}|y!jMIK0dlZXEGs}XmEx&_iKa;%ld|QzLQ0LJ+b5CXSGc?1 zlzhnyxF^D~JWQ;aLasr_By4yvLIh16+8K_xMOINX*F)4zCD-%msSWr{nRXvOUzE>+ z!bUT8*#BxFGKuJ0WFnTbnE6EmIz&E93@%=egOmk~EV1beC@&$E_sbF#QfU7Rv$@Sy zJ=;&@Ob^CR#naJ}xpz7^9U;KMpRfqwUyA0TqaF@XUrNs3-cp6Qn!IFeaR`qYI7lSV zNa_br`3Y)- z=DGeIZ0@{yPR`sozeV3z0#dWUPgLAA7?HTNV3fFUldlUBU)@yLB@#6YYUJ2L^El{N zf#{ShB**)Q84=>lLX734dJ?{6huk2VH=}$>wkW!9c8@xg9)%qnmS*@EiGOf9lECN>NT?1zFa` z^7-?MF{W9NdfS&F3cwOM2CvIrGq)}$AJ4HhoBRUt_1yzRZruRUy1dHNtGGH2Jp+;m z3FP(4FWs6yT|t2dEpx%DAW^gOobj#dy#Dq#mywIrmNPYx;?wN#?W+&z73t`6c!{R9 zRidVblqvx~Q*%WxrK1nxC4AQ$asfG9c&(kH_lgud2ZxGB*B&<1Ts;>qylSh2|9U6f zNU`aIwVNEzuvv4eUra<{}vQ9Nu%e6Zm=^qX!cDB zqUlyY6_W+4f<@h?V9|E#J9|L1gLf<`rWqaABML_H)}(i z>OGmE;e11z3uYnW(>q8-(E5t*Y-8Tf<=++IK3m=06xv~@=UVUfRUjsd*1P@o4*03L zbH)~Oh!w7qo|)LOyuYZwbA>4R*(TH68C%Bm7-aE&&arhE0mLDDx80)va}~BQG4{Uo zG`~FeX;>Nt(~`KasJZ{wnlyl84xefkaFM^SSp3fe=wyh4`)tm%yVRsGZL@3j2d)*wFCcPvx^*6g^F{E0xIZ zhZd-nr=AYBrK1W)F9ZK6s!&;s`;vx&X-T4aJ?goKk9NDFN640RyQ218w_rDAT@4Q+ zxkd33N^N`FcznHDPhoO(L4TfG=2KHOto*O{6i5xnOB(DMgcWBNhU*)A z>1~^6emOzDBjv=|H{-;>*UHdRM2Rz}LPb%FC$ig~Vx`xdrrjSd^`2`v-sZj+D`0!&+lHv zXW{EF%SPzXUc$n_o6%w2n>om$GV%%dJ(?h;xajvIwBjMX24b+UdrL7+xHa@ou`~*m zecz812TsmKG^P1bD6P0_%CR=_!=<Vmg9{wgOfP$4B8|eA77!o1T3~9S_GR zG4ycWjika*(e{T>71Dz9r0xky-eguK^4_BcLTbwS06eu(8Y@NaJ9d$m7Ys#HSd?)B zt*7t1O3y#)qsca8yjv-X5tP9TLrjHK)ytG_Yp0sQyz0Zo;M8kZZ5}iQHt{ zg$9E}5ly4#WRXjZZS5DvbuABK)JKWJ{-L+%NlH?|%$HjAm3cDdkH{I_JBi+Jc#3{QddPnc)2Q+F!FP;}K& zRPk#uE_1fhx|H$WBKL7Nx262@KUhYV*z)&o zmH?+YOBP3e@D?XOb&AjzJy;?tX6M!Tx`R$`g{N3 zXQG5ugGc|d%~aXo!$yh+|2YsLwET0Vgd>%ZDEV@PsiPd-{N+|-r5N`W9n<{bE7B*$ z|8nu50dM$5LiI(|nQi|noo$6E{DzM2zW>H1&V4gLmPVNtQ2cF}D7#83mr94Ok~yMM`uASq>eX=dE%jTa zo=&Ey_42oi)Y9_rw9-f4W$I%`BZ9?NHo-L2@4lyF zJ$L&LWL=5<4h6Fn2n{(T<+-Cjc(CiC839lsdP59j%T+)mQ1A+4D*|0N0JlLkQz&@4 z@&FXLEM0s29}AnhS`xG=(Co=Ddj2m++iYd&KSIi2uNRwz#DBZ35GM7OXVsm_T7US7 zH_J0_GWi$3G*zH{6(TgFZ27?;U#6HoO=GCSP-nQk8Z?5~Tn<5UR;$HZ+GaJ5-0UNp9pbyEhBN z4RWeUEo#QH=s^Z9VUi4LfaWI1MS0r?LxE2P?3FtkQ8yFro_o+3{n<6SGX zCF9Dm+g9A&mUtF|f6m$@8)dwUS7dPMDocS)N3c^0r2d2qqqOI~dtogxCX#0+$?2 z0cTA%s(%H4V~w%w4WnXMmYEh(QSOdmi>`;|<)HDgY+^SX_COGihSL751~FGW_-Czn zKYZe-a%V=1rXBJQ=3~uR!Jcql+||CI?Siud*lPTfA5UANZV(bE0?Ja@Or_lhXL@1~ za`YeGuqlBmvHizUOre{Gwp_?-iLMJ-_mb1I{;w?L<{ck1&mI^VceNa7O<{EEbz@8E zn`RKAv~;H5MM1_QZi5HY8O9k@4ug-52j?=Gzic&=8AEHQ;l!{yQ5W#DgBbd69eb(EvdQSk@D$Ev zlf)=x3`(oCvKEj&i4|J-=?fV_!kHfoWy_HG2fvO6 zJo#Uq?CJ(6U^)vwY!IBo+^;y)xiOZt?HJA`B9b?az`JDsEI+t^1WVO#rJ;E^i_uz( z)KJBPGNggE*nTWE-+~%=?|p=Ga4W=m!(pw3!Ks$?lkU~Lc;1% z%pdmUvR~s+va^5{M@-$a^4L6Vr0Z(skNeHEGBP~h#oRB=XP74en`a={1Nn?1gO<@O z83v7J_bTNVB$H2o)G;hZT=_#V92kuU3HEW!7yQObk4z5u#aPnNC=RlxA4b&y&^8LS z^3HKC4ng#I7T*EeT^es-JEVTL#&&d62PG331u*waWN9v}uz3{%%h18MCbUjM_ zFO+a}C8>z@+N$yNrni5O9Mx9xA`N6`y)X9;(0fZt9WG+y4HLN85=|)Z#tx-WiPl;%z{l!DOokBgke;nFdvHlb}Xah@9ALwQ%Vi{KXnF6*7l#6a{`Wg zV+I4$1eG(H%Q32+T|vU@xEXmmcb1&Lb{4~H!mi!$50pNbe`waFi)XVG9NRdXneXb~ zn9Uw=;l=ybpz^MtBekC_x6WaBXf|+T6!5va)X9OK2TYf4ic-v;wv#jQcH9jZaCmlY^iEdk!0v`;m%XKbshGf zf@uromLb!FmfhyAoWQ&F!w-ZBV90(df6=%lq4wn#f8Cc5{iIXc+9%} z4i*V@ZOjWLOc_J_jpwvjlgulzwOV9E z1tN5mk_kR^%b;OGa^X`YM=r(6kq|vw$|l_;Lf&^Q$TrX71W81u6GX~gmtZMu2+5%2 zL^)cf8pdG8s&R;NWur`}PD`wtOrxCrOxO}8_@5HxZ$`+&sedeGR8n>-N*lp}WK>h&8gRtcF%8TBGUWLgrXwm>-HvMEFx zF%L6471pPIF+dF411ToU)IJ z(Q^o(M6W{BuC$^HPW#}HwhTUWut-{!jbzKXx6XGg0jUdvs;xwtCRij7hgk_>CC%7L zy^S)ALVqwo1jU7xS0h4{4oy38GwvZ}7%6+Hph&d41#2i(+KxSlH;I}{aYQ-3SINP1?EgiMLa(EV`$rWRJCYEx>^2Xat?p_0Xjx(!}V z;j!@PB58!txwJygVm98z1hrJ4_jVZTQTDJL3o%O=#c8!m*iV%|VQ}$}=)6TOWkp)W zv~w7rT}wb`Z7G|Q>{O^kT8-EpI@84d#R#SGBwB#_3dP7&&^3cH`NU+S+j(rb35fnKv#Fg-a4V1;yr-&#SAF{b3j!u>N@ ze-~QmxtOqTrF0W888fjn69FMrEFaOGQ^lUcKYdoQ=ZxM6Cj2GR95E1F?Y1eVi2`6> zwe+33oUP|-I`xOitJ&?wY2fSz=8zT2-e}CyfcK&BWDQ%dv(K}&%z0sgky||Yt&wMu z5o_2O?A6hI0(J|5V{2IC7=_zO4k}Sd=?fy_d_|v;W?O~aPD+tJ8Zfu4wbV!%N+j#i zb1{xx4-M;Cm=8&xX8%ZO#JgKL5(!086uEVd#!EskgE=c+%aT>t1d0vEP%x1MIHVJu zLgucINfhxCJ`94_vQ#tJV*T}Y+2hT%EK{%A2sAOJE5~Yx(l#h4I|W2}->7?_p-^-) z+Srn{EC~){$*K)8%9kqM@8b^QzV})CYU_ET&M*o8s^yetZVH*aGBeT@Ex z3TFH*vd^4b*ev6CDE}4N@=`Cjc#8~7Fs~83Hn3_J$)FU6P6eSW79`x@@U3i#&WCg{ z53k?Gf^>f#FFmsnfBX{ysT*063Bko1nZpH{#*H#YeHH&CJN1!~c23f3lqGd3mC}eh zspb$y1Uy;CrggfiKDT!mmLA-VSblOl+n_J#?9Z{rb2C$t$!VgD%?!U^fc$!N6}N3> zk*@f6p;g4e=?)05mku+IFEhBFOpg*zy5s=y!Cp{Ix4 zx>P=A?~pE;&z+J9=iJGLn}Kyb`qG^=8dGK?#HXQ=P;wVby^i6u7EHSJ_F06gWi^`6 zl}O>qGq`Kq-_-&nJ(}*8p?KkVJPf&;O}b7L;He#GBv zwlIHOEnRgL{gjn-Lt2TM#}>j2cKr$^BX0{E&=p|v-05VstL1iP6eRDC-D=?7srML+ zLW`ZJ(f0t*+rdqrIlx!4qlrHM&-U@se-+@@GpDtkAwvBn6k;D?KQ~hWg~kC_9+8KLUGlfK5??sBlMWGR zIbm>RC(G*q;QU>TE=#)v*og^4PIMl9BNGnSZB|zs|NBSTI1`f*d!#?cs*RQKtH;<% zGq&JpV!IGiGPVlBHzq&rdkk3j+spK?tYN}l_83C`>t5-(ChlXVPlsUhK6VrCYyRm# zXxqo?JCh02AOSGwaaKWnDFYdBk`QsI)UK1K@(C%Bk4@#nT-GSV`t&E|T@Zbm(y4Ng zzGrwQ55-wp{uv0@o@BF41VkpF;|kQ8#pXFM9nuC(7g(@so~0rIGoK=ZC!M)_pK{|E z8awnaEGP^Wp^F@uwN%Tz!T)34vdIQILSlhw;#U7HL|6 zyw5qlUrGaNvSjc-AZ@W$_JH{Z%-@s2<^v269{x#$yk}ilN`vPO7oMeqJmta&C-8=( zgLr2!v{^=6vRhR%%QPV)gUOT4inmEAU~SE8imAthgNzb&w6*xcv{l?sen5t<^NwH3 zEp;~5|C;O|H2zv%0z2z{S8#u;k+{0ErbAY!WG1&X3U$9ih*7^W1bNaj4EimbtFzui zzh$H3X6!$V7r`Ef?XsS_9)UVK3w@SknLxctgn%wJwIx}ks>7gm_^ z3d@9jM`VAAaMLSzhWXABxub4FI@f0*@a2oHCT#~w1f;*jMz|C4zT-&54_}g}yKb{c zBGw#p67iK|T@vw;mpdk6&T%IZcOGXCqD1`|BGMHXl8)}H+)&eKMyK2wx#+aPtz##Y zYIqk-ZPXue#7D2nD{4j2z4aRVnMNeE`e88sJ-kiY`a3q>$X$3Aod({oyScfS7V!y& zvna)0E6fvHiGlG943%r}=!ml5oytdz9@XP^c}dgux)cHZnC`YW=uQfW76Hw_XQ6IJ z8$}6fd-4#s26R0nsWqrxu3`iJ_g!c#`M(%`*z&jd?bRrqdro4C6aS&KrE+lNVQ=VYpgB&ei|A zQM@qQIs)qdB(IpIo)8N^j*uRRjvuCLB*Uh^uw0ow+Ebo9VdixrA>#6ZuAT|ILekg) zM?Yd?X_)DgN!WFft|2j3Y%(-`%qGEt518@84hkytbZneu{|9s?L1C#(ys@{knq;Sr zF$M~{J=WbH(h`P&zHp_Lb(7~TNfP>@3rWbaaw7@e&2~qLiI{;ycjJw%k{U})9GCnt zOUAbSe=Q(F%yiKc9n%1sK4BZKcl}WIS61g{)y;0oJ^#S8oL~ISpxB=NCmQ!{e?j-_ z+}~M_zIrrD(fC6>k($Z3Q65e>-217E!)FLIqCR6~rj=tFXCjR7;hFH!XKWyDGFCJi zdq@#|{j#*)k^={S&Pq)k)C8b`va*AvY`@~Vlqnyf_y6V>tjxF+8%)6jZNxx|ka};Z zoxMrC@HD!OO)}06j<00!E{ELv7245F?JU@!z;#AcpkfzP-tDY|Z-N2I?op||ArhuX$kzVN**xeG}c`uJ?%_OjG&jRbU~vH2!OJ0r3Yyk)$|>9i|oxKq4itl z(ZTp^DE&@mvXZ}ny~!4!zwo1O@y#r@>w5!vUcAO;=-XD_n^UaesvB{yLg-#83hMiD zj6WaB=celrYJaGePeP2agx?&H8 z&+Wi#!(_-b$o1yqt(`k-%{_S%ocSXrLil^gKvs9F#(HqNg;Tat?gr*ZMq(B&U!qXc zYE;Sk$xhc^98QPVe6$zJ?QMW(H`?Ge{F3{hZXT&Gsn4q9#%4D* z_lNoMNR=pGZu(Uf$oJ(hBK5xaMMmM%_&}F!>pHX8sP-hrpGI&x zTabB1luHc#Bl&2FeLUE0+yk~pl6Jtp@?8-Mt;xm}-}%QWEGxd4?4p!I^tgv z#ml$`D2wG31ZxuE4UfiB4k4hVWWa@3?u})16hq)zEFUBfQRJ0I zS%2>5t^u_x4qte(KOfLF5k&ctFt|U?$smG$w19qqNMHV212~=G)30j4+<3mmq^Ggb z24BV#4V50-5;(?=RK|04rRqo`UuPO5LCsI%YqTDo1>A2(GGAfrg{Pf1{DrbAyt!V4 zp*MY>D6dkLAFeT*E4ow{AEg99Q=C-OimH%b8i&txt^`lg=uE=kxzXJK?lu{(-=uOp z-%w8G7NzkzBQ8K^o;%XHCw4GjaL`4Dk0HH`nr(0*os0_L?Z@IB?=Wu{nYt;1XUje8 zI|WHvgg>wd_C}WKwow{~8xCBR{NG8LWJuwmOg_(qr93H&%HkA7(~|6ixG&UX@rRx7 z`%D8)58}fS5L?9KAU&I}(8|ZNx%sQ1tihZjqWs~AaQ$Gpak3mdg#SXTzc7R!(8>pf z@@CUamBTauDR6w4oCyJ>!0a4;8)8SvL(A8rr|CVM4_1IYulzyfFoHj$cDXXb1?;az zattvPypYX&!6TRSi&D`%0~OBBQ5-)5%;o1?YtJhcaqw^+e_Sgs$>)!o@V1VYLN6Cv zHClrmho3xiuN`CfQ`$g$->M>6HF24_$O|vWES+-%y*DqIjBiku!KuO_qju;6=)CqZnW3xvz{a zPR)BQOB0>{_@>= zBe8`%KgF5Cp=v4dyitrgpXo;h7^(r1ckx>*xPJ#*Aurn&E#r%D0gBa~v7GB)8fy20 z;c&Eyn{Tzgsd9DO=u?S!`mO0G*ib3s1gsbb@95buxWcq*jPTa2;6dF81S043 z&kL)#o^^O*c+trzqD05q_(x(B1J$cIn(u1vr{FcJ8v|_+u_4$oRnmCIZ&-nK-??>q=U~OFH0KOyeN$_uY5zdnv%@w;vxjbElj+bLO-;bM8C4 z#_ia@y4G%F9%2v6uRD{61q*vvzS1+7)AJ->C7=FcR;K{9RNpCFEgkLDpp?>Xb6V!K z)MNK8wzksS@7^?)`c{PCa(#y5TT5u1X!2R9VlmUzS1k3V zMG__le50{+&UXVo7yGTH-hK<@v#+T4?~SEKuPAZJ--=s#unU}+$o<5+fPu0xMqCP* zg(aGDQQ#!4WD?DRLuHRJF+OOaZt^ohM0uJ#BA9rK$WR_^o#1lt92G`I$UJrYPaz9s zKa!G`P@%QEfN3jpgPqSacX6mkj&6>CFoK*LR)kMlz`n5YG}g}|lEZ1Ql#&^?n0R-Q z>WeTXRP>?@KZ1C%$gY-cFG zkwMtyuE>7)d_6Kr?bxlSMY#3sseuU0`m4VNGQE6)5 zB~fwA1JZs((7U7js9RJ*J_*DEl=Njz;Z8%GN>SbyITfJnPjyOnny!kOwY?B^!7_(d zf`EDhD*G{SQJd`|#z*%-;K!rG;PPtbM*<~ZhsbBT*geQ93&*3=Rf!3UO%pQ)xpqx; zN_I-6Sq0VDq$Vsjw;Ro4ovU;a3l#Unh9JnTv9!vqu~Fh`ED7J%*kGty%G`CZuu5!< zbAzkp%vBtVbya#{Rgg-Vm*YrZd>%&>^o%D84vHtJp7Ft=Hl9fJLVRC*ei=`D@nw7{ zJea}*a0aUynwSuS4XYEV`P+n0v2%z?H~+T@X?pBM%NSpgok++QB=&`X#Azr9zbZD_BLCxSHywtI4ej6Uk}rD(pdNq;~F2BOIRY6)H}A zXM#gln2TI!hTJsPD-9c2FMsjHhh~xO?IKEgxvIk#%eJR`jk5CK_M<`KY%k;ug(W}3 z*XAQGqA4BsN2%KUZn*L)UP`AGy_6m)8a{Hjb)yx`(!a@PTAwJ< zaKr`Q)A~>?z0Wv9?|<(TDR!PzOI><7ML3bI`g;jJijP1JN=vP+aLx21t3oHXv9t(3XMu zose2^vIqATYX*+Ay)@}xa0Cce1T!=cZb?D3k_-`bzW zpz?5mjcZuEwN_YSqO2y^S?n7WB-RXa)-@(!T@US@{WY%M!&$iHXjM=%hDD;PF^Q5K zKWjgwvLUCZz9NQ3zL*o>nXXE?bqi@NdAYbr@Nf=GcUZ!aTy&7ADHNJ=Sb#(O(81Kc zYajC!<%8qp_9&3i{{c3K#ff8s(O$xy6)aSQ4+-$sm=oO|TNyn+#N&=3(H3Ml z>4p~V<2>{ulwx>3^0rbe$tQeehhzD9hG{J$GvvEMR4cEyj3nnk;d`&qxsoAhWtOr) zv2-*#B5e_~=8<6GGbXbwByqQVMd1`@Q8fl#QD7^O3y+N<=OAh9L->3{KJyFej5y+; zUkZ?4glD0FXv&hN6-KwZgh0`!$R@s7#i80VN{dM)K3ANNIP3}Gp$@Z^Pcn)A<4918 z#-YV??6hYbLV%igS+FpVN8{334~tkco*ct%;}sHHB`0m2ML6GrgG9nDNv%vBAvWKF z3ibyl$J%jl%j9$iysd*3<`^tZudIWDdKB5+Q_QXG@6@R_ z7QJ&?qqesMq+*6i%$wc^$#ibUAaQoOyB!igPA_N|iHaE&a%_Qd9Mmk}zM^<0Iq27J z2^J@2qUS}*DonVS>~(->+>X*Y8KUr^Nu8=tj1C>J949$yhY zJHo&lQgOJ5g|lhSn!$+3*a-pRi?iKZp-t%=8*OUl_S0z7IF}AVar4Od&X&(7=M6E; zg4C;@uc83v-rOXL=N~clC?j9XvOa^{2XW=zXpvnNEt<*}8G03!N1+2jJdSHo0KK>1 z2Zt^f7fw^Uh#8B0M8&PA^lyvKXm1znM$T4SPE>@5iy1-nOZI9Nspy`#iiX=3iHZu+ zvqZ}i6_>SAD!MDK!hPvpJCFl}>usa8ULj(0p8)aHZToaJznlsZu9b^~d4&ybh*-b; zgaH!qAE}Ru+g-%Q6?8t%y(d8g^z{*+t#IraUTqei-Hsxs(dpI9mEp_*Vt93|Xjo-Z zv6->hPgGU=iRM+GYBfIM(j8yx-mczxNqKwq_bT2hK(#Z`@Q&*5b#G6twhnDrNl6vYoz8|F@1H>6W-!Nv9I~x;g96N6@B+oqhgQ)1GB8v# zH;@?8T~us*#VQ%FYYrb{6n2sOY?1iO!{~#EJpTa z)%Z+%;a>TiEiToQLz(shmE6TWf?`AT=K9D=EY&=6zkHU8!k45NFW|Z2gGYO*mB(K4 z3r$6Rj4p`glPha+Tu~+%RwQ<>K$Z8{!44PnI9a3)7u52^?buCOR?Q=b?vof~fR=fD zd6!HDLDBLg5rrC0Jk>`nrEXVBhE>Ww$6gBQP(KEWe{3}_>O_T)@Y{B@{Y5Rgz6ISX z{9d3P`=2hVZih_~?|J#<_7_ql%)5wR9Ck<94P_rp-6$BjRM`aDkq2uMZ*ZwTA za$m8omYfhYtZ<+=Gke-e0d~)~@47!u)b2OwYOP=wWx6_e<=6IS*!9dkFi}?{C*L0` z7QSJ^Gp0S%+ykTht!GVpfB%{s6{Lejc+|ZyQ&&UKe|sZWSEKY9Awv4O+2cyo3KK^U z>W?bc?s%AOz5B8^d$ccNP?L_~qT)lEjKS94SaD)zkha0+J_!}O--?y*xOd{@2T>yN zND1199^%BY08!ZJg6y{S@N4qvD@vLu-sddx-kv8eG?K2|de~QsdgQ&m*`TrsI+;cD zJEVlD=fPglqWqmlwE4Ckjl}sjYwsuVz4^#y?7iXLcko&8-dp(ee4n1j-+xOF9a@T6 zAh=JtptZPHD?(}?7A;bbYNKWpV;uBQAB=;B zmpj9uY~~8p#TYc(akL6Y<3wSweJBQ_8;&}Ql8+R-g!@C0ImO5*q;~%-MC|-{3gRj) zk3w$6YcuOeP9e3rCV!E73~>|R9UrEWd!BxIBKP>qvX!)M+Ubchs0f8?5YZhYnU4}< zG@Trdt-C&*j6P6!=R^wW$i63rU=NDeLvubG4~GXKQ5sIsfoj*MR(OX;GIV%OhX58b zMf0BnR7f+*whFn=)(`ztQP%&%fl(YRZug%%fv9w;YDZ#c5=+QC+zSmx{!c#5^vSDu=E6*PF3ikXwkW-_iw{9 zo2`|WrVMoxssAkNaA7v3KJ%-z8(xek`I=VK^0jd_FMk6Lt9kC`R+28({i}mjz;VtJ z#KCKB;^T`p!TPQf#*0oIux$|gkoiKkGb6c|XC*XM`aOVM&Ws$*RswKv?QJ}BhI~tw zxag;JQSq()>9OcSD4gxgdW+A#%|!SXe+&bqUPf$W5WryBF-3* zhDBFdck$r&CIdyJdOY}jt)a4}8_N=p{3j71H2x<`!jXDP6#p>L&{2-w{lj{Fr5N@j z#dQAsBdM68pX~f=z&pY$_&5yJ=+&QSEB9PU7c+izgDsvcNbLMMHNnQ{9g2Yl%uN(t zDUrbLZ)+n&!G9^>_uGF%#i{>B%hDhN+C{$xijrSQ7gK5PFJ!E!l=_>i_~n-%^)2~V zrM^aezx3-YwN!T1S~`D~Tzh5gVBD4>UpB%p)w{n@ADxHG^is2Q+3fQ3VOARFmGKBLn!Ge{?7 ztAkmVCS>mfGYVL5jpE)A9>R>_L9aXul!vhXM*RUNP23lfLzy2mg|Kj)Y%7dKY>`{W zf~m`Tgxwg!{PEApP{~JWr|HfOpJ4tAlBd>%yF=4@EH(6?O@>AnZh?zE7`X*$r!BB0 zOf3d?BG-hfniH6dn0C$`s>4~1(oJoPfn%M~S4ZUrhkBqF+sh)QD(!R(Y_Tx1zBs-U z(gx01SSZexwvhS9*_4yA0KH|5C^poH6>U)poN(hIQ0UBE+TNm_(L9g#CYmk5Ke;iq zMe2shJt|3J6O{H)IMEr;E(gDNgX&mWd7ZQ~a-v+)kvRJ$Eql}EfB(@-S`H_!GS~Lk z1x-l|Z;Y;&AB=1AOwmbKN-EQ?w&2_wczmo)W&IH}mV@EzR7UO{x@}>QoyIZ^TDK;R z4I(u0kD(rq1pZ#j&`Ynt!&f%`r+A(Rr1xT_`hLPP(tp!W=!NyTYJyX}m{~R&QN`1` z&D!EgJx)^uCwsHWIC-m%?feWj5!=XKz^M!pDW!}@+(p*1^?3UtR#*?S?r^jZ9ssSU zS*zVi&y-U=l*xwKvF(?cj4l)QTnPibNhnLONv2*7+d8y_f?t2;cx9qaD=<>9X#gX3 zpFR+;2+ipx*f@|SYggW|djN~Dwic?blu`rz2U;62Fb3acF^f2c?LD$tBBj>A$Sl$= zN&_ZQKu9)ArVJZ6rZue9@03El@4L*wpTM&3x{V3q*A2eSv2@_wldM?mrr79lSG*$;LO#)w4tFy;=X zq0)ns1D+X5IvjO@>}kT|cz0+Xgz9?pFgu?iY&eT)h2zc**Kr(DKf}s#6!?SUJSh)# z%eX&0lg9$=_@Qwb;@^}<)JGKX&Ie7#rb@&!I-fXCsT+lc=ukder+?o(ngvnNL5n`1 z1X_DPr4*42m80>jeR&kSTPeR}tr|Xts;Iwp6q%}D#%OpnjAbUs8B2begeu=xj%5@L zAUyG1F*XPQatoLRuE@?xIk|v3QkgDYssH){@;sFZ!MbP+1$8xd>MxC+FYrPu$hLZ4 z>e^lFjf8TA1BHxEZ`KJMjEz+=@D>op7D*-bRuSe_JYU2D5I-y%QZN)&^C@O{{!q#& z^YvIpao^g?&L@r6?fjTqSb}xui7ox1X%=}};5nPkhWH7rw}JCp?ql9iHG%c9V+gz# z%?Ha)U)P=9D8c%vm)5=(P|8zqTTWMGP8ecdx zAGzxA6k43zwB$4R&ZkO!iw>3lrYXpUv!*hN$4TB)+^o;0k{6`6+GY}oIn$)mOrDlx z&%M)B1`38{lUVmI2~G)gDI`svI-$vnyF$)%;u_6si$;bp&%h{O($HJZQ;^@Y~SA{b>m+w)Vh^?y#(ajZUz~Ss{3ZQLs#H9qK-d}F)#ec10~TJv zKX(FNG+r_z)-9PcFNQK%%;?WRqaW?X164O-LqjF3|#} zmFz*iHo`ojI5@VFm9;T-cc@+^9e{S{|8f-@rknrdX(aZ@JJ=+{Oz6Tv!w%*qD;-?7 zK)Cx(B!QLH9;jlrh-+h&fod&-=!~`ig=5hbjE2d*Y&9}ja~De=uo$zG%Q5MZa+Jd{ zsb?uZ=ize+CiH}3fOr-DorQ0U4BBe}_KAfrSFl{Jn^v=ET|KS{uiOG5 zb`9&Lt0pOt-6kOxGUWi_6OrC_wXEEn&&3IQ>v{uHRFspH`-Tu&BYg-7@ z$)jDiB}`%Xy(p_z-lbbYLfS`j3h=Te16PWF{NdnzEZ(ph^am_<2!-7*UECu4b3-g6 zwcOGBSs_k_FoL1aIySbgmBAD1TJ>rNy@*&Ydw{Lf7But!u;!8VOi6S_juv~yp&?k$ z;&GmC8(4}R`t9ioIrPO1($pT`z)FonW%OXmgB@x-3FY#GtS2IXP)5OwhuA_#P~up= z8TdvPg55eqYRyLJ8a}>}jlu5UY-9tB06T_I`>=$7d9K0YVn7HqJ=A~S8_-xU zLo4l37AgfBVzofvwHNK(F=5b~q##vzlo<#!9$_UNZC5)oDYz zXHN-V-5@oWJW-r)V53`&?cu@JU&ia;i+c3hmu;mn z@=VcugayUf{6l3U^}!W=8IM%7pAx~L&q%pN1no7M#BJ|0Y#I_?qSf#h#4~XlE9@9I zcQ~|-O}S~@C|tJZS@xunTBsl`Q1zS)Vze)J>gq7O0Fv@m&$FCXU|oD(Mo`*=)t(7N z=CuXgkU<7$Hm^P2n7Ew{GcY6Z_o3~qTwe)&Ut+f!F_yW*aO@>kg{vZ~jel-N$zJVT62$kL-lU9rY&(o zm1HxhI3@D%D=b{2*3Yl7Tjj~mi;kb8Cv*GDC>as3-6))RRi0!i8NW?^Qu#^C|*O=`y7pb5Jzs53kD@=liO%Whay-?8o+QRh8Qe zpJC8!+9};NqU6La@3H_ZRiVpSyl3|i`33f0+@Qlk67u0$iZ&`P%b_!DUTbSo7V_TT zG042=Z;TQFDV};`PE>o`A@vR}#%A{9Ixc;#I~f0=>rgHP{>UXDgJ-~$FQpf!_`oUW zQU6V8(rtw_qJ_s<1ng~+QD~LpQ}xUao?o$CSagoa46CVp_8hbSit}bX8N|@$e<5-) z8jINQ+I+rUyp|{^*}Gadfeq$@27~Dvk`hG~h?!quss#EC8dm2nvjvpI=Gs|)wjonO zaen)c3bhQOJh=KVc{NqnWPy>*Y^Z!UkT&k3j3|vcHwuzl*hsi^5fe;p#{iUGnI(#| z4EQ(0&&VmpAn*DsF5y}Cz`xm?1Y2*#bhZ;z){kE1ppFfeSCZrTZ|%rI?vS1wWJM@3 zwmmA8A=WAZ<`~w#znRfToSt3#eU8y+u;Adlkm$WP6uSh2a zGY%5q$Q3qT4}vrkP8kH_|I3tvy(9eag*+Q%q9HfT9)@4zg-i0CX#hW6?b|3`=(9 zZm`{lhp1Yq&GA0)k1ia6NT4{1CcDm?K_l_vy6T9R@5XP@cJ(_SPGrUA1I<`?1rqxlZ@Z@y2r#MPp7)JWSlMluU-Hu}NRR~m=IY!MT0Ucl=x!U)} zaFTc^^WqfSwazrQ{YkFFK{M`>bYa$ebDIkzPwvPm*?+)(k!E{T*-lYXlnS-}+?XkF z+@HUJE7}mi_uFaqqCifmjZXw}3W4B95P@(hkPqxYTOSY7G2M5;e+bjTZ=t*ZaX<$L zW49i7!MeT&x4!^GbTi=h9(-j72x11E&6~k|KrLbVk5?(kfUz(ol2f+IO-V5FMj%Y+ z$!&L*#2aG!@Mw)SF!3S^_m>VjFHYu(c6*2io$JkGv{Q#GW2Ah_Oa;nZgjZrnzECN} zKu9cKs1|p|k{Bwraij|5LKAGxP6rWEjBpj;S;t#V%G0bLi8-a!e;>z7I@nMtKp083 z6QI;YZad3hG+v4JAUd{KJYxG%BA;kbT1iQg)$UUMe3E4U39^zlGccJ`77q$V5^PE4 zZb}(HFiPf_)1(N?aQ(K#>6W6NoYEUk8TSPT{2jM`*VzNC@G}h_vglBeazhShm8Zci`v<62O@{# zJqOwl2y-5PT_x&KJ*NH7hi?V<9! z)~ba=_)eu9135$a4#U`=hsv>*VH~gz{z-zbhw(eGTsoXncurPb9nRM)pHj-#=+9{Wk@6=2 zu8g(=wrvbg$3|RF6kHu6?S1K3ej2OrPZG>4;7Vl10&ayI&pFDpn+he95{~gjyx3aZ zP{f~AQQscOBVb*zRBI=T2?4pHL*w`zI$uK5bz)aspvPSKqm>IVGFFb>Mdf%&GD-Es zTL=xRN`t0bh#@63G!m}x)C7)^1qCmci5j}wC))99$|Oz!G?@UnY!ask>8q2p{YMYX z8QdmweHtiY2U@(}SHqRQaI-2~$eoFHb@OE2U$-jF+fp2#9?>$zN%5|fj|?#P9c>1# z!R8w0lyDcgV-hz(SvNjUHcBt@_o|bf z@i&*?hf<#7pavY(o7lc%1!AR~aVW22(_oZ`@Hu>)rs3N7C5$$HnxNeuC0Gm>=5s%y zg8T)H04%lk3mCGxRN&7N1h&3xtQCzfP85Y(0_aXjc-2nEU!*Yk>V63U31VFB>05QK z*UyW1oppgy)1meTRgh62NdP=CIK$`)-V=L}%jWo{&$4G&w&TT3v< zTd^c)Udm_T3Xlh7ny2;mcgvmTLkrp~ZJ*S8KEp{LF#BBj?OU79z#MG)8Up{sfcX%5 zk2$M&cPLpdy`$1qd;}a^PFx6Yu0jWuui%c!Tt?QBSYBLV#S%yU{dTV9JX-G`uLrMT zaCN10Z4Jng*+#tRa{)wYPzPeW?G7CyzjG&FYb6kU2uALB<6LjJuS$9(k5+NR_oeVn z70dA!;h5Audzs(HM;;Jr15 ihqgL0cPf6Tws19%a9E(qB{JB4U^Q1h+r~X0p7?*p#@}85 diff --git a/oi_scraper/.env.example b/oi_scraper/.env.example new file mode 100644 index 0000000..5a47651 --- /dev/null +++ b/oi_scraper/.env.example @@ -0,0 +1,27 @@ +# CME Group QuikStrike Login Credentials +CME_USERNAME=your_username_here +CME_PASSWORD=your_password_here + +# Product Configuration +# Gold (XAUUSD/COMEX Gold - OG|GC): pid=40 +# Default product for XAUUSD trading +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool + +# Alternative products: +# SOFR (3M SOFR): https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +# Silver: https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool + +# Gold Price Source (investing.com) +INVESTING_URL=https://www.investing.com/commodities/gold + +# Output Settings +CSV_OUTPUT_PATH=./oi_data.csv +TOP_N_STRIKES=3 + +# Scraping Settings +HEADLESS=false # Set to true for production +TIMEOUT_SECONDS=30 +RETRY_ATTEMPTS=3 + +# Logging +LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR \ No newline at end of file diff --git a/oi_scraper/.gitignore b/oi_scraper/.gitignore new file mode 100644 index 0000000..4b3fd0c --- /dev/null +++ b/oi_scraper/.gitignore @@ -0,0 +1,31 @@ +# Python cache +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +venv/ +env/ +ENV/ + +# Environment variables +.env + +# Output files +*.csv +*.png +*.log + +# Session data +cookies.json + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/oi_scraper/README.md b/oi_scraper/README.md new file mode 100644 index 0000000..508f647 --- /dev/null +++ b/oi_scraper/README.md @@ -0,0 +1,178 @@ +# CME OI Scraper + +Python scraper to pull Open Interest data from CME Group QuikStrike and current gold price from investing.com. + +## What It Extracts + +1. **OI Levels (from CME QuikStrike):** + - Top 3 CALL strikes by OI volume + - Top 3 PUT strikes by OI volume + +2. **Gold Price (from investing.com):** + - Current gold futures price (e.g., 4345.50) + +## Prerequisites + +- Python 3.9+ +- CME Group QuikStrike account with login credentials + +## Installation + +1. Copy environment variables: +```bash +cp .env.example .env +``` + +2. Edit `.env` and add your CME credentials: +```bash +CME_USERNAME=your_username +CME_PASSWORD=your_password +``` + +3. Install dependencies: +```bash +pip install -r requirements.txt +playwright install chromium +``` + +## Usage + +### Basic Scraping + +```bash +python main.py +``` + +This will: +- Login to CME QuikStrike +- Navigate to OI Heatmap +- Extract top 3 CALL and PUT strikes by OI volume +- Scrape current gold price from investing.com +- Export to `oi_data.csv` + +### Session Persistence + +The scraper automatically saves your login session to `cookies.json`. This means: + +- **First run**: Logs in with your credentials, saves cookies +- **Subsequent runs**: Uses saved cookies if session is still valid +- **Session expired**: Automatically logs in again and saves new cookies + +Benefits for scheduled runs: +- Faster execution (skips login when session is valid) +- Reduces login attempts to CME servers +- CME sessions typically last several days/weeks + +To force a fresh login, delete `cookies.json`: +```bash +rm cookies.json +``` + +### Output Format + +The CSV output is compatible with the EA's `LoadOIFromCSV()` and `LoadFuturePriceFromCSV()` functions: + +```csv +Type,Strike,OI +CALL,4345,155398 +CALL,4350,229137 +CALL,4360,90649 +PUT,4300,227936 +PUT,4290,270135 +PUT,4280,65839 + +[Price] +FuturePrice,4345.50 +``` + +**Note:** The `[Price]` section contains the current gold futures price scraped from investing.com. The EA reads this value for Delta calculation. + +## Configuration + +Edit `.env` to customize: + +- `PRODUCT_URL` - QuikStrike product page URL (requires login) +- `TOP_N_STRIKES` - Number of top strikes to export (default: 3) +- `HEADLESS` - Run browser in headless mode (default: false for debugging) +- `CSV_OUTPUT_PATH` - Output CSV file path +- `TIMEOUT_SECONDS` - Page load timeout + +### Available Products + +**Gold (XAUUSD/COMEX Gold - OG|GC):** +``` +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool +``` + +**Silver:** +``` +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool +``` + +**SOFR (3M SOFR):** +``` +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +``` + +**Note:** You must be logged in to access QuikStrike data. The scraper will automatically login using credentials from `.env`. + +## Integration with EA + +The EA reads OI data from CSV when `InpOISource = OI_SOURCE_CSV_FILE`. + +Place the generated `oi_data.csv` in MetaTrader's `MQL5/Files` directory. + +## Scheduling + +Use cron or Windows Task Scheduler to run periodically: + +```bash +# Run every hour +0 * * * * cd /path/to/oi_scraper && python main.py +``` + +## Troubleshooting + +**Login fails:** +- Verify credentials in `.env` +- Check if CME requires 2FA +- Set `HEADLESS=false` to see what's happening +- Check screenshots: `login_failed.png`, `login_error.png`, `login_success.png` + +**No data extracted:** +- Check if table structure changed +- Increase `TIMEOUT_SECONDS` +- Check logs for detailed errors +- Screenshot saved as `login_debug.png` or `login_failed.png` + +**Login page selectors changed:** +- If the scraper can't find username/password inputs, CME may have updated their login page +- Update the selectors in `login_to_cme()` function in `main.py`: + ```python + # Example: update to match current CME login form + page.fill('input[id="username"]', CME_USERNAME) + page.fill('input[id="password"]', CME_PASSWORD) + page.click('button[type="submit"]') + ``` + +**Browser issues:** +- Install Chromium dependencies: `playwright install chromium` +- Try different browser: Change `p.chromium.launch()` to `p.firefox.launch()` + +## Notes + +- The scraper targets the OI Heatmap table structure +- Only exports top N strikes by OI volume +- Login session is not persisted (login each run) +- Cookies could be saved for faster subsequent runs + +### Finding Product IDs + +To find product IDs for other instruments: +1. Visit https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html +2. Login to your CME account +3. Select a product from the "Products" menu +4. The URL will update with the `pid` parameter +5. Copy that URL to your `.env` file + +Example: `https://www.cmegroup.com/tools-information/quikstrike/open-interest-heatmap.html?pid=40` (Gold) \ No newline at end of file diff --git a/oi_scraper/WINDOWS_SETUP.md b/oi_scraper/WINDOWS_SETUP.md new file mode 100644 index 0000000..9e06a8a --- /dev/null +++ b/oi_scraper/WINDOWS_SETUP.md @@ -0,0 +1,658 @@ +# CME OI Scraper - Windows Setup Guide + +Complete guide for setting up and running the CME OI scraper on Windows with automatic daily updates. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Installation](#installation) +- [Configuration](#configuration) +- [Manual Testing](#manual-testing) +- [Automatic Daily Updates](#automatic-daily-updates) +- [MetaTrader 5 Integration](#metatrader-5-integration) +- [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites + +### Required Software + +1. **Python 3.9 or higher** + - Download: https://www.python.org/downloads/ + - During installation: ✅ Check "Add Python to PATH" + +2. **CME Group QuikStrike Account** + - Free account required: https://www.cmegroup.com/ + - Register for QuikStrike access + - Save your username and password + +3. **MetaTrader 5** (for EA integration) + - Download: https://www.metatrader5.com/ + - Install on your Windows machine + +### Verify Python Installation + +```cmd +python --version +``` + +Expected output: `Python 3.9.x` or higher + +If not found, install Python or use `py` or `python3` commands. + +--- + +## Installation + +### Step 1: Navigate to Scraper Directory + +Open Command Prompt (cmd) and navigate: + +```cmd +cd C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper +``` + +Replace `YourUsername` with your actual Windows username. + +### Step 2: Create Environment File + +```cmd +copy .env.example .env +``` + +### Step 3: Edit .env File + +Open `.env` with Notepad: + +```cmd +notepad .env +``` + +Update with your credentials: + +```env +# CME Group QuikStrike Login Credentials +CME_USERNAME=your_actual_username_here +CME_PASSWORD=your_actual_password_here + +# Product Configuration (Gold) +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool + +# Output Settings +CSV_OUTPUT_PATH=./oi_data.csv +TOP_N_STRIKES=3 + +# Scraping Settings +HEADLESS=false +TIMEOUT_SECONDS=30 +RETRY_ATTEMPTS=3 + +# Logging +LOG_LEVEL=INFO +``` + +**Save and close** (Ctrl+S, then Alt+F4). + +### Step 4: Install Python Dependencies + +```cmd +pip install -r requirements.txt +``` + +Expected output: Successfully installed playwright, python-dotenv, pandas + +### Step 5: Install Playwright Browser + +```cmd +playwright install chromium +``` + +Expected output: Downloading Chromium... [progress bar] + +--- + +## Configuration + +### Available Products + +**Gold (XAUUSD/COMEX Gold - OG|GC):** +```env +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool +``` + +**Silver:** +```env +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=41&viewitemid=IntegratedOpenInterestTool +``` + +**SOFR (3M SOFR):** +```env +PRODUCT_URL=https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=476&viewitemid=IntegratedOpenInterestTool +``` + +### Configuration Options + +| Setting | Description | Default | +|----------|-------------|---------| +| `TOP_N_STRIKES` | Number of top strikes to export | 3 | +| `HEADLESS` | Run browser without window (true/false) | false | +| `TIMEOUT_SECONDS` | Page load timeout in seconds | 30 | +| `CSV_OUTPUT_PATH` | Output CSV file path | ./oi_data.csv | +| `LOG_LEVEL` | DEBUG, INFO, WARNING, ERROR | INFO | + +--- + +## Manual Testing + +### Run Scraper Manually + +```cmd +python main.py +``` + +Expected output: +``` +INFO:__main__:Cookies loaded from file +INFO:__main__:Using existing session (cookies) +INFO:__main__:Navigating to OI Heatmap: https://... +INFO:__main__:Extracting OI data from Gold matrix table... +INFO:__main__:Extracted 6 OI levels +INFO:__main__:Exported OI data to ./oi_data.csv +``` + +### Check Output + +**1. Verify CSV created:** + +```cmd +dir oi_data.csv +``` + +**2. View CSV content:** + +```cmd +notepad oi_data.csv +``` + +Expected format: +```csv +Type,Strike,OI +CALL,4400,6193 +CALL,4300,3826 +CALL,4350,1983 +PUT,4400,5559 +PUT,4300,2988 +PUT,4350,1214 +``` + +### Check Logs + +```cmd +type scraper.log +``` + +Or view in Notepad: + +```cmd +notepad scraper.log +``` + +--- + +## Automatic Daily Updates + +### Option 1: Windows Task Scheduler (Recommended) + +#### Step 1: Create Batch File Wrapper + +Create `run_scraper.bat` in the oi_scraper directory: + +```cmd +@echo off +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +echo Starting CME OI Scraper at %date% %time% >> scraper.log +echo ---------------------------------------- >> scraper.log + +python main.py >> scraper.log 2>&1 + +if %ERRORLEVEL% EQU 0 ( + echo %date% %time%: Scraper completed successfully >> scraper.log +) else ( + echo %date% %time%: Scraper failed with error %ERRORLEVEL% >> scraper.log +) + +echo ---------------------------------------- >> scraper.log +``` + +Replace `YourUsername` with your actual username. + +#### Step 2: Open Task Scheduler + +Press `Win + R`, type `taskschd.msc`, press Enter + +Or: Start → Windows Administrative Tools → Task Scheduler + +#### Step 3: Create Task + +1. Click **"Create Basic Task"** on right sidebar +2. **Name:** `CME OI Scraper - Daily` +3. **Description:** `Update OI data from CME QuikStrike every day at 9 AM` +4. Click **Next** + +#### Step 4: Set Trigger + +1. **Trigger:** Select "Daily" +2. **Start date:** Today's date +3. **Start time:** 9:00:00 AM (or your preferred time) +4. Click **Next** + +#### Step 5: Set Action + +1. **Action:** Select "Start a program" +2. **Program/script:** + ``` + C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.bat + ``` +3. **Start in (optional):** + ``` + C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + ``` +4. Click **Next** + +#### Step 6: Finish + +1. Review settings +2. Check "Open the Properties dialog for this task when I click Finish" +3. Click **Finish** + +#### Step 7: Configure Advanced Settings (Optional) + +In the Properties dialog: + +- **General tab:** + - ✅ Run whether user is logged on or not + - ✅ Do not store password (if using Windows authentication) + - ✅ Run with highest privileges + +- **Conditions tab:** + - ✅ Start the task only if the computer is on AC power + - ✅ Stop if the computer switches to battery power + - ✅ Wake the computer to run this task + +- **Settings tab:** + - ✅ Allow task to be run on demand + - ❌ Stop the task if it runs longer than: 30 minutes + - ✅ If the task fails, restart every: 5 minutes (up to 3 times) + +Click **OK** to save settings. + +#### Step 8: Test Task + +1. In Task Scheduler, find "CME OI Scraper - Daily" +2. Right-click → **Run** +3. Check `scraper.log` after a minute: + ```cmd + type scraper.log + ``` + +--- + +### Option 2: PowerShell Script (Advanced) + +#### Step 1: Create PowerShell Script + +Save as `run_scraper.ps1`: + +```powershell +# Script configuration +$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper" +$logFile = "$scriptPath\scraper.log" +$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + +# Navigate to script directory +cd $scriptPath + +try { + # Run Python scraper + Write-Output "$timestamp: Starting CME OI Scraper" | Add-Content $logFile + & python main.py *>> $logFile 2>&1 + + # Check if CSV was created + if (Test-Path "oi_data.csv") { + $fileInfo = Get-Item "oi_data.csv" + Write-Output "$timestamp: Scraper completed successfully (CSV updated: $($fileInfo.LastWriteTime))" | Add-Content $logFile + } else { + Write-Output "$timestamp: WARNING - CSV file not created" | Add-Content $logFile + } +} catch { + $errorMsg = $_.Exception.Message + Write-Output "$timestamp: ERROR - $errorMsg" | Add-Content $logFile + exit 1 +} +``` + +#### Step 2: Update Task Scheduler to Use PowerShell + +Same steps as Option 1, but: + +- **Program/script:** `powershell.exe` +- **Add arguments:** + ``` + -ExecutionPolicy Bypass -File "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper\run_scraper.ps1" + ``` + +--- + +## MetaTrader 5 Integration + +### Find MT5 Files Directory + +MT5 data directory location: + +``` +C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Terminal_ID]\MQL5\Files\ +``` + +**To find your Terminal_ID:** + +1. Open MT5 +2. Click **File** → **Open Data Folder** +3. Navigate to `Terminal\[Your_Terminal_ID]\MQL5\Files\` + +### Update Batch File to Copy to MT5 + +Edit `run_scraper.bat`: + +```cmd +@echo off +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +echo Starting CME OI Scraper at %date% %time% >> scraper.log +echo ---------------------------------------- >> scraper.log + +python main.py >> scraper.log 2>&1 + +if %ERRORLEVEL% EQU 0 ( + if exist oi_data.csv ( + echo Copying OI data to MT5... >> scraper.log + copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" + + if %ERRORLEVEL% EQU 0 ( + echo %date% %time%: Scraper completed - OI data copied to MT5 >> scraper.log + ) else ( + echo %date% %time%: ERROR - Failed to copy to MT5 >> scraper.log + ) + ) else ( + echo %date% %time%: ERROR - oi_data.csv not found >> scraper.log + ) +) else ( + echo %date% %time%: ERROR - Scraper failed with error %ERRORLEVEL% >> scraper.log +) + +echo ---------------------------------------- >> scraper.log +``` + +Replace `[Your_Terminal_ID]` with your actual MT5 terminal ID. + +### Update EA Configuration + +In your EA (`OI_MeanReversion_Pro_XAUUSD_A.mq5`), set: + +```mql5 +input ENUM_OI_SOURCE InpOISource = OI_SOURCE_CSV_FILE; // Load from CSV file +``` + +The EA will automatically read `oi_data.csv` from its Files directory. + +--- + +## Troubleshooting + +### Python Not Found + +**Error:** `'python' is not recognized as an internal or external command` + +**Solutions:** + +1. Use full path to Python: + ```cmd + C:\Users\YourUsername\AppData\Local\Programs\Python\Python312\python.exe main.py + ``` + +2. Use `py` launcher: + ```cmd + py main.py + ``` + +3. Reinstall Python with "Add to PATH" option + +### Module Import Errors + +**Error:** `ModuleNotFoundError: No module named 'playwright'` + +**Solution:** +```cmd +pip install -r requirements.txt +``` + +### Login Fails + +**Error:** `Login failed - still on login page` + +**Solutions:** + +1. Check credentials in `.env` file: + ```cmd + notepad .env + ``` + +2. Check login screenshots: + - `login_failed.png` - Shows login page + - `login_error.png` - Shows error during login + - `login_success.png` - Confirms successful login + +3. Manually test login at: https://www.cmegroup.com/ + +4. Check if 2FA is required (CME may require additional authentication) + +### No Data Extracted + +**Warning:** `No CALL OI data extracted` or `No PUT OI data extracted` + +**Solutions:** + +1. Check if you're logged in: + - Delete `cookies.json` to force fresh login + - Run scraper manually with `HEADLESS=false` in `.env` + +2. Check if page structure changed: + - View screenshots to see actual page content + - Check if Gold product URL is correct + +3. Increase timeout: + ```env + TIMEOUT_SECONDS=60 + ``` + +### Task Not Running + +**Issue:** Task Scheduler doesn't execute the task + +**Solutions:** + +1. Check task history: + - Task Scheduler → Right-click task → Properties → History tab + - Look for errors in the log + +2. Test manually: + - Right-click task → Run + - Check `scraper.log` for output + +3. Check account permissions: + - Ensure task is set to run with your Windows account + - Check "Run whether user is logged on or not" + +4. Check Windows Event Viewer: + - Event Viewer → Windows Logs → Application + - Look for Task Scheduler errors + +### Session Expiration + +**Issue:** Session expires after some time + +**Solution:** +The scraper will automatically re-login when cookies expire. No manual action needed. + +To force fresh login: +```cmd +del cookies.json +``` + +### Check Logs + +**View recent logs:** + +```cmd +type scraper.log | more +``` + +**View last 20 lines:** + +```cmd +powershell "Get-Content scraper.log -Tail 20" +``` + +**Search for errors:** + +```cmd +findstr /C:"ERROR" scraper.log +``` + +### Verify CSV Output + +**Check if CSV is valid:** + +```cmd +python -c "import pandas as pd; print(pd.read_csv('oi_data.csv'))" +``` + +**Check file size:** + +```cmd +dir oi_data.csv +``` + +--- + +## Advanced Options + +### Run Multiple Times Per Day + +**Edit Task Scheduler Trigger:** + +1. Open task properties → Triggers tab +2. Edit existing trigger → Click "New" to add additional +3. Set different times: + - 9:00 AM + - 12:00 PM + - 3:00 PM + - 6:00 PM + +### Run on Market Days Only + +**Create separate batch file:** + +```cmd +@echo off +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +REM Check if today is weekday (1=Monday, 5=Friday) +for /f "skip=1 tokens=*" %%a in ('wmic path win32_localtime get dayofweek /value') do set DAY=%%a + +if %DAY% LSS 1 goto END +if %DAY% GTR 5 goto END + +REM Run scraper +python main.py >> scraper.log 2>&1 + +:END +``` + +### Email Notifications + +**Use PowerShell to send email on completion:** + +```powershell +# Add to run_scraper.ps1 at the end +$smtpServer = "smtp.gmail.com" +$smtpPort = 587 +$smtpUser = "your_email@gmail.com" +$smtpPass = "your_password" +$from = "CME OI Scraper " +$to = "your_email@gmail.com" +$subject = "CME OI Scraper - %date%" + +if ($errorOccurred) { + $body = "CME OI Scraper failed. Check logs for details." +} else { + $body = "CME OI Scraper completed successfully.`n`nUpdated files:`n- oi_data.csv" +} + +$message = New-Object System.Net.Mail.MailMessage $from, $to +$message.Subject = $subject +$message.Body = $body + +$smtp = New-Object System.Net.Mail.SmtpClient $smtpServer, $smtpPort +$smtp.EnableSsl = $true +$smtp.Credentials = New-Object System.Net.NetworkCredential $smtpUser, $smtpPass + +$smtp.Send($message) +``` + +--- + +## Summary + +**Quick Start Checklist:** + +- [ ] Python 3.9+ installed +- [ ] CME QuikStrike account created +- [ ] `.env` file configured with credentials +- [ ] Dependencies installed (`pip install -r requirements.txt`) +- [ ] Playwright browser installed (`playwright install chromium`) +- [ ] Manual test successful (`python main.py`) +- [ ] `oi_data.csv` created and valid +- [ ] Task Scheduler task created +- [ ] Task tested manually +- [ ] CSV copied to MT5 Files directory +- [ ] EA configured to use CSV file + +**Daily Workflow:** + +1. Task Scheduler runs at 9:00 AM +2. Batch file executes Python scraper +3. Scraper logs in with saved cookies (or fresh login) +4. OI data extracted and saved to `oi_data.csv` +5. CSV copied to MT5 Files directory +6. EA reads updated OI data +7. EA uses new OI levels for trading + +--- + +## Support + +For issues or questions: + +1. Check `scraper.log` for detailed error messages +2. Review screenshots (login_failed.png, login_error.png) +3. Verify `.env` configuration +4. Test manually without Task Scheduler +5. Check Windows Event Viewer for system errors + +--- + +**Last Updated:** January 4, 2026 +**Version:** 1.0 +**Platform:** Windows 10/11 diff --git a/oi_scraper/main.py b/oi_scraper/main.py new file mode 100644 index 0000000..a5ba0e6 --- /dev/null +++ b/oi_scraper/main.py @@ -0,0 +1,257 @@ +import os +import logging +import json +from playwright.sync_api import sync_playwright +from dotenv import load_dotenv +import pandas as pd + +load_dotenv() + +# Configuration +CME_USERNAME = os.getenv("CME_USERNAME") +CME_PASSWORD = os.getenv("CME_PASSWORD") +PRODUCT_URL = os.getenv( + "PRODUCT_URL", + "https://cmegroup.quikstrike.net/User/QuikStrikeView.aspx?pid=40&viewitemid=IntegratedOpenInterestTool", +) +INVESTING_URL = os.getenv("INVESTING_URL", "https://www.investing.com/commodities/gold") +CSV_OUTPUT_PATH = os.getenv("CSV_OUTPUT_PATH", "./oi_data.csv") +TOP_N_STRIKES = int(os.getenv("TOP_N_STRIKES", "3")) +HEADLESS = os.getenv("HEADLESS", "false").lower() == "true" +TIMEOUT_SECONDS = int(os.getenv("TIMEOUT_SECONDS", "30")) +RETRY_ATTEMPTS = int(os.getenv("RETRY_ATTEMPTS", "3")) +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") +COOKIE_FILE = "./cookies.json" + +logging.basicConfig(level=getattr(logging, LOG_LEVEL)) +logger = logging.getLogger(__name__) + + +def save_cookies(context): + cookies = context.cookies() + with open(COOKIE_FILE, "w") as f: + json.dump(cookies, f) + logger.info("Cookies saved to file") + + +def load_cookies(context): + if os.path.exists(COOKIE_FILE): + with open(COOKIE_FILE, "r") as f: + cookies = json.load(f) + context.add_cookies(cookies) + logger.info("Cookies loaded from file") + return True + return False + + +def is_logged_in(page): + page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + return "login" not in page.url.lower() + + +def login_to_cme(page): + logger.info("Attempting to login to CME QuikStrike...") + + page.goto( + "https://www.cmegroup.com/account/login.html", timeout=TIMEOUT_SECONDS * 1000 + ) + + try: + page.fill('input[name="username"]', CME_USERNAME) + page.fill('input[name="password"]', CME_PASSWORD) + page.click('button[type="submit"]') + + page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + + if "login" in page.url.lower(): + logger.error("Login failed - still on login page") + page.screenshot(path="login_failed.png") + return False + + logger.info("Login successful") + page.screenshot(path="login_success.png") + return True + + except Exception as e: + logger.error(f"Login error: {e}") + page.screenshot(path="login_error.png") + return False + + +def navigate_to_oi_heatmap(page): + logger.info(f"Navigating to OI Heatmap: {PRODUCT_URL}") + page.goto(PRODUCT_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("networkidle", timeout=TIMEOUT_SECONDS * 1000) + + +def extract_oi_data(page): + logger.info("Extracting OI data from Gold matrix table...") + + call_levels = [] + put_levels = [] + + table = page.locator("table.grid-thm").first + rows = table.locator("tbody tr").all() + + for row in rows: + try: + cells = row.locator("td").all() + if len(cells) < 3: + continue + + strike_cell = cells[0].text_content().strip() + if not strike_cell or not strike_cell.replace(".", "").isdigit(): + continue + + strike = float(strike_cell) + + cells_with_data = cells[2:] + + for i in range(0, len(cells_with_data), 2): + if i + 1 >= len(cells_with_data): + break + + call_cell = cells_with_data[i] + put_cell = cells_with_data[i + 1] + + call_text = call_cell.text_content().strip() + put_text = put_cell.text_content().strip() + + if call_text and call_text.replace(",", "").isdigit(): + call_oi = int(call_text.replace(",", "")) + call_levels.append( + {"Type": "CALL", "Strike": strike, "OI": call_oi} + ) + + if put_text and put_text.replace(",", "").isdigit(): + put_oi = int(put_text.replace(",", "")) + put_levels.append({"Type": "PUT", "Strike": strike, "OI": put_oi}) + + except Exception as e: + logger.warning(f"Error parsing row: {e}") + continue + + if not call_levels: + logger.warning("No CALL OI data extracted") + if not put_levels: + logger.warning("No PUT OI data extracted") + + call_df = ( + pd.DataFrame(call_levels).nlargest(TOP_N_STRIKES, "OI") + if call_levels + else pd.DataFrame() + ) + put_df = ( + pd.DataFrame(put_levels).nlargest(TOP_N_STRIKES, "OI") + if put_levels + else pd.DataFrame() + ) + + result_df = pd.concat([call_df, put_df], ignore_index=True) + + logger.info(f"Extracted {len(result_df)} OI levels") + return result_df + + +def scrape_investing_gold_price(page): + logger.info(f"Scraping gold price from: {INVESTING_URL}") + + try: + page.goto(INVESTING_URL, timeout=TIMEOUT_SECONDS * 1000) + page.wait_for_load_state("domcontentloaded", timeout=TIMEOUT_SECONDS * 1000) + + price_locator = page.locator('div[data-test="instrument-price-last"]') + + if price_locator.count() > 0: + price_text = price_locator.text_content().strip() + price_text = price_text.replace(",", "") + price = float(price_text) + logger.info(f"Extracted gold price: {price}") + return price + else: + logger.warning("Price element not found, trying alternative selector") + alt_locator = page.locator(".text-5xl\\/9") + if alt_locator.count() > 0: + price_text = alt_locator.text_content().strip() + price_text = price_text.replace(",", "") + price = float(price_text) + logger.info(f"Extracted gold price (alt): {price}") + return price + + logger.warning("Could not extract gold price") + return 0.0 + + except Exception as e: + logger.error(f"Error scraping gold price: {e}") + return 0.0 + + +def export_to_csv(df, future_price=0.0): + output_path = CSV_OUTPUT_PATH + + with open(output_path, "w") as f: + df.to_csv(f, index=False) + f.write("\n[Price]\n") + f.write(f"FuturePrice,{future_price}\n") + + logger.info(f"Exported OI data and price to {output_path}") + + +def run_scraper(): + if not CME_USERNAME or not CME_PASSWORD: + logger.error("Missing CME_USERNAME or CME_PASSWORD in .env file") + return + + future_price = 0.0 + + for attempt in range(RETRY_ATTEMPTS): + try: + with sync_playwright() as p: + browser = p.chromium.launch(headless=HEADLESS) + context = browser.new_context() + page = context.new_page() + + loaded_cookies = load_cookies(context) + page2 = context.new_page() + + if loaded_cookies and is_logged_in(page2): + logger.info("Using existing session (cookies)") + else: + logger.info("No valid session found, logging in...") + if not login_to_cme(page): + browser.close() + if attempt < RETRY_ATTEMPTS - 1: + logger.info( + f"Retrying... Attempt {attempt + 2}/{RETRY_ATTEMPTS}" + ) + continue + else: + logger.error("All login attempts failed") + return + save_cookies(context) + + navigate_to_oi_heatmap(page) + oi_data = extract_oi_data(page) + + if not oi_data.empty: + logger.info("Extracting gold price from investing.com...") + future_price = scrape_investing_gold_price(page) + + export_to_csv(oi_data, future_price) + else: + logger.warning("No OI data extracted") + + browser.close() + break + + except Exception as e: + logger.error(f"Scraper error (attempt {attempt + 1}): {e}") + if attempt < RETRY_ATTEMPTS - 1: + logger.info(f"Retrying... Attempt {attempt + 2}/{RETRY_ATTEMPTS}") + else: + logger.error("All attempts failed") + + +if __name__ == "__main__": + run_scraper() diff --git a/oi_scraper/requirements.txt b/oi_scraper/requirements.txt new file mode 100644 index 0000000..d29d1ce --- /dev/null +++ b/oi_scraper/requirements.txt @@ -0,0 +1,3 @@ +playwright>=1.40.0 +python-dotenv>=1.0.0 +pandas>=2.2.0 diff --git a/oi_scraper/run_scraper.bat b/oi_scraper/run_scraper.bat new file mode 100644 index 0000000..7356d73 --- /dev/null +++ b/oi_scraper/run_scraper.bat @@ -0,0 +1,47 @@ +@echo off +REM ========================================== +REM CME OI Scraper - Automatic Daily Runner +REM ========================================== + +REM Change to script directory +cd /d C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper + +echo ========================================== +echo CME OI Scraper - Daily Update +echo ========================================== +echo Started at: %date% %time% +echo ========================================== >> scraper.log + +REM Run Python scraper +python main.py >> scraper.log 2>&1 + +REM Check if scraper succeeded +if %ERRORLEVEL% EQU 0 ( + echo [%date% %time%] Scraper completed successfully >> scraper.log + + REM Check if CSV file was created + if exist oi_data.csv ( + echo [%date% %time%] CSV file created successfully >> scraper.log + + REM Copy to MetaTrader 5 Files directory + REM Update this path to your actual MT5 directory + copy oi_data.csv "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" + + if %ERRORLEVEL% EQU 0 ( + echo [%date% %time%] CSV copied to MT5 Files directory >> scraper.log + ) else ( + echo [%date% %time%] ERROR: Failed to copy CSV to MT5 directory >> scraper.log + ) + ) else ( + echo [%date% %time%] WARNING: oi_data.csv not found >> scraper.log + ) +) else ( + echo [%date% %time%] ERROR: Scraper failed with error code %ERRORLEVEL% >> scraper.log +) + +echo ========================================== +echo Completed at: %date% %time% +echo ========================================== + +REM Keep window open for 5 seconds to see any errors +timeout /t 5 diff --git a/oi_scraper/run_scraper.ps1 b/oi_scraper/run_scraper.ps1 new file mode 100644 index 0000000..28f3e11 --- /dev/null +++ b/oi_scraper/run_scraper.ps1 @@ -0,0 +1,77 @@ +# CME OI Scraper - PowerShell Script +# Copy this file to: run_scraper.ps1 + +# ========================================== +# Configuration +# ========================================== +$scriptPath = "C:\Users\YourUsername\Gitea\MeanRevisionEA\oi_scraper" +$logFile = "$scriptPath\scraper.log" +$csvFile = "$scriptPath\oi_data.csv" +$mt5Path = "C:\Users\YourUsername\AppData\Roaming\MetaQuotes\Terminal\[Your_Terminal_ID]\MQL5\Files\oi_data.csv" + +# ========================================== +# Helper Functions +# ========================================== +function Write-Log { + param([string]$message) + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] $message" + Write-Output $logEntry | Add-Content $logFile + Write-Host $logEntry +} + +# ========================================== +# Main Script +# ========================================== + +# Navigate to script directory +cd $scriptPath + +Write-Log "==========================================" +Write-Log "CME OI Scraper - Daily Update" +Write-Log "==========================================" + +try { + # Run Python scraper + Write-Log "Starting Python scraper..." + & python main.py *>> $logFile 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -eq 0) { + Write-Log "Python scraper completed successfully" + + # Check if CSV was created + if (Test-Path $csvFile)) { + $fileInfo = Get-Item $csvFile + Write-Log "CSV file found (Last modified: $($fileInfo.LastWriteTime))" + + # Copy to MT5 directory + Write-Log "Copying CSV to MetaTrader 5 Files directory..." + + try { + Copy-Item -Path $csvFile -Destination $mt5Path -Force + Write-Log "CSV successfully copied to MT5 directory" + + # Verify copy + if (Test-Path $mt5Path)) { + Write-Log "Verified: MT5 CSV file exists" + } else { + Write-Log "ERROR: MT5 CSV file not found after copy" + } + } catch { + Write-Log "ERROR: Failed to copy to MT5 directory - $_" + } + } else { + Write-Log "WARNING: CSV file not found after scraper execution" + } + } else { + Write-Log "ERROR: Python scraper failed with exit code $exitCode" + } +} catch { + Write-Log "ERROR: Script failed - $($_.Exception.Message)" + exit 1 +} + +Write-Log "==========================================" +Write-Log "Script completed" +Write-Log "=========================================="