From 4a3d5d21d1818d3e077e644240b87d825a351454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Thu, 4 Jun 2026 09:08:51 +0300 Subject: [PATCH] fix: review findings - OUTPUT_DIR centralised, no hardcoded vwts.ru, normal imports --- vwts_scraper/__main__.py | 4 +--- .../__pycache__/__init__.cpython-312.pyc | Bin 238 -> 238 bytes .../__pycache__/__main__.cpython-312.pyc | Bin 2503 -> 2571 bytes .../__pycache__/config.cpython-312.pyc | Bin 0 -> 473 bytes .../__pycache__/fetch.cpython-312.pyc | Bin 0 -> 2204 bytes .../__pycache__/output.cpython-312.pyc | Bin 0 -> 2534 bytes .../__pycache__/paginate.cpython-312.pyc | Bin 0 -> 4039 bytes .../__pycache__/parse.cpython-312.pyc | Bin 0 -> 2810 bytes vwts_scraper/__pycache__/run.cpython-312.pyc | Bin 0 -> 5289 bytes .../__pycache__/state.cpython-312.pyc | Bin 0 -> 1449 bytes vwts_scraper/config.py | 2 ++ vwts_scraper/output.py | 4 +--- vwts_scraper/run.py | 11 +++++++---- vwts_scraper/state.py | 4 +--- 14 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 vwts_scraper/__pycache__/config.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/fetch.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/output.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/paginate.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/parse.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/run.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/state.cpython-312.pyc diff --git a/vwts_scraper/__main__.py b/vwts_scraper/__main__.py index 3771ff0..b1009dd 100644 --- a/vwts_scraper/__main__.py +++ b/vwts_scraper/__main__.py @@ -8,10 +8,8 @@ """ import sys, logging -from pathlib import Path +from .config import OUTPUT_DIR -# Настройка логирования -OUTPUT_DIR = Path("vwts_data") OUTPUT_DIR.mkdir(parents=True, exist_ok=True) logging.basicConfig( diff --git a/vwts_scraper/__pycache__/__init__.cpython-312.pyc b/vwts_scraper/__pycache__/__init__.cpython-312.pyc index dbe588661f66b252ba044b7ab7579f0f878a26c8..89813b7f9c9184c6bce07757c014efe09f958f9c 100644 GIT binary patch delta 18 YcmaFI_>Ph5G%qg~0}#jyPvm+6058e}i2wiq delta 18 YcmaFI_>Ph5G%qg~0}!+cP2_q405Qn~?*IS* diff --git a/vwts_scraper/__pycache__/__main__.cpython-312.pyc b/vwts_scraper/__pycache__/__main__.cpython-312.pyc index 8c3124c4277b23781c94292f76f19d9a627e855a..c60ae33fc6a21a863d1d5020d0378d4242fff308 100644 GIT binary patch delta 388 zcmX>u+%3X)nwOW00SLYdD`p9?P2`hcY}lyI%E&T-v50T79;1d;v0w^+Dr1%mND(SX z5s(B5lra`5F)^g_rwFcQgvc{8fcZj`^B9%+1rVYs0x80i7cj~*3QRu0D9J4XlSvVo z{DjewQFQVPCPihjRN*XrRC_?Y6ohK=$yrRA#uBNlvw@<7RfF6JR4F-^6=eQoZ)Wbv zpP0mfwsA2#GD?HoI60qLf>CC32lFaMfhzX0@{(e`qSBWj&y-qBHepraP=Ja}j$n=2 zyqz_eiE-m(ZjKJ^%gmBD*||E5`z<;xnrkN%USgM+;khDseZi;4T&NpCWci06#mtWAVmxe zj0|ADz~lmEx`(V3Z&uLn>pI3`8|DnIb3&?(CQWwnRMy!HKy^SlQ6xE#*+6lLxvZ0Um=zf%K_u85 zGiJ-l+|1mQtC$TLr8lo(Ud6~%YA`vCRb}!d)+pAOAa8Ac!y3%QxM;EiM~BQMX33lE zTph;!7M&K&wG#?2u}j?K;Jv`-aD~Hh^Jfk=Mxmo-9BvE@#~3u+w3&}-F#<^d1FS-6 diff --git a/vwts_scraper/__pycache__/config.cpython-312.pyc b/vwts_scraper/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6af11cc013d4d6c2f914beee9ceab27f0ca1e142 GIT binary patch literal 473 zcmX@j%ge<81lI%=vliP+?}Gso+>pkdqpon&h2XqHk(!u4inf z$$pDHK0YNsIX?atTXBAJc52Bj!Q#}Cl+?7u(wvf#%-q!c(vn-OF0MX~k+;}GJbhjL zLql%yhxiA0ItRxGxCX_$dHT5CV)t-$ba4#|Udiwo=st#D`uZ99xvBbjiK(eM`gx^E zsm1y}KEAr4p893wCB^Z@$wi3;sYUw9`FUxX>3Ri~zc_4i^HWN5QtgTafcinfS8NF+ zJ}@&fGTvm6e9XZ6fsKJtvVrY7gTzG!i5Wsy7-Sys3pKFb;1_RTzab>r!2f_n;DS)V P2WBR2rbhN6KA>&@p}v4I literal 0 HcmV?d00001 diff --git a/vwts_scraper/__pycache__/fetch.cpython-312.pyc b/vwts_scraper/__pycache__/fetch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2f6b0f64773dcf87ce359ceecbcd45971884152 GIT binary patch literal 2204 zcmah~YfKbZ6u$G?*(bX^*KQSuP!l!^3*wJTj0VfLf?!oJshgP1x^scu?gQ`4$Rp5z zHW4ccn3_myNby(Ks_B~gAWcg(wW&@2%*ZC-l%|F@Y4}y5CT;AW-n+}V+R%EFx#!$- z&pG#=$9LyPhrnpVHjhSgbZ zL_5!nWFNf_hE8q2797vu2r`yw6MNj@>Y%$dOJZ z)}B;)mPvG8M(3LRt&N9uzN5L#|MsB{o!{ecZ1Nv$*BJ%HbQ6NIf)e4?R1jCdfhPWomOURP&@dT&h@LrcTw z*22*ij5-20^_F&0*q2Penq>k4j4NJ_#9l{XcLOM+lgr)JTf6R;v#AAc7Z+Sma zWg;J9jmT9p{Yi8kSi1D_bXJjoT~J7G3)vgoRIlTMmdP842!<5W>e`6T#`@whg;dh8 zR6kK?@e6TAJE4B5riB64SKlBklP6LkMO&I06Vxg7hLC+pzR-rXlj=?UOnj}*HV9s) z&XVivoD_qw2N5fsL!m?$)OiU?QY4wsnJy?J6WOsza!6LBU^pp4orxv8biOx)5|Kog z?n0r61cRMP6wE0hk{;|G_Q|@%-xr2GGI7R1gp$b1A_8-rO6U~QDJYQ;aoU8ZHNNg- z9QqO=2xGp4auiCw*4DP_L(RV9y|NUP!YI@Ok*^cV;qIEAendJxI}!Lxn{Zl(!Edbj zRLe-qn37>j7ac_tb!Xmr1PVCMD78X6ES^P|`@HR}ZSsvP?g`tRt8TP$!RZ(kx{ArJT3(Zq()H=xslHTHUB5l!YIxv&)lhV5PT#8HWIWaQdE2G7)UKH=x6!@2 zdk21~zteX+s_s9iu5Zt{Iu>0e=dEY0lUt_?rw?2#{b)zVRkh$+w_tb7+sie3dB!d* z77NpbpO#%LOO?$P*WS0+uDJ1apOgd6e5!e*S#?#X!s*KN)>L;YpccHIVYe^XoPRyB z0M{#z0F5KEJID8)-m7}*(iQ2(YnAEJv^Y~YQ>m`onX&I$w7bSzPPa@P8*lrd?e7&1 z8$6XZ;dEVU zx_6P1Ge@Ouxhk6xJf=WNmUL4%ndppkWg98mGl)F)x&^mTD>j2jHwS}>P#j`!q=|LD zJ=bN3{DXDA8-_%PupgyI49U33BpGGPdkH+o3IGXsdTQPgPl`$m?nE_siE9p|FrHQz zilQEX*55(N&tSt(!1)Who@S+y(Ua~-a+eXV=1 zn_xMWNxKzPCXz7;Pc57Gj|zUI6HHmVb<$L%N&9mziG+6qKlYHO{__S?CHiOIxwZ+- zrhU;p-#O=dobT=YDG=}?fPZ%nWfC5Q{zeaN@V7P(mtk`iVKj>{gV`)I%ea`GWdU

&c0;B7<0@3Rt|sG_}o2aAq@I?MF7T>V3lF!;Gy`Zq)u& z^vKNdW7DHiy9Rtq_NwzEd(F8VwLb!ZU)#5wcY*!n@e@-o&nPDnuPTS8rV}mk(a)%9 z$6^eTaMDsOEvLesKasS~d`5kNLvV{Ek>8lNOa<7J#BZu8OVjg;nJvz3p7P0@N(4(U zXelBTbki~kZzbnUBA6Dul7bvO0U352d=Cjr<6jmQJ!KDO`rrz)XD1Od2HI!qiBJhN z0Cg8o13={#*ag;=;7b97Qe29i0NyfFX0br;0ef0lq}ZY_x=`lY`6zQ`-V)mPnWt@R zUEXNn47GU+v}ss+Us@C`x85jA8I^npElOo61D0(CEMxhm_Xc=Qfk9;n`=~#zFaGy< z9Jt20^0F0d@_6_T)qWdymw?oc z1=Mz!{Xm43_2G+7v72XBa@kD8T1;&o5_A)-NkRw zE-oYuOZgtaXvWm@*(d$H=2V=w?Ay*#)Oj1ke`A0A<<>sdZf4?hdrnE|#k^(Esv_RB zmQ@ukkJSZ(7B^uv46UhIS(R`_BO8;5*lJ=THKl=}rKiazH?M)2-doA6;O#d7ADKCRq}SR(kJ2I(^i9 zO2=A$?hqXOYThgws**HQn)aCS9xHajD^V93zxc@~%|Ml;rk?074eWhxF12FK1qxEoKb$I35^`3$1cURu5Z|kofSb6<^51=chdMHxw4Bw9o zUd^s#ujKDVMxDs$dSu@}Lvn|wa_CV6$=%DZJJL|)#X2v%bNKw><;mJu;zZL?+siG?b_d+91XrzXn(C+JFPAI<0ffcKh4UXw}{t>+W^`~O?oFfj_hVF_x z8$2&})%~H%M=l=u@pMB(((va&*9O9~+U;vG&-?B?$H`w}DEoe=@{u}&f?3f5DK zV#Gj(h)|bv&{@qAhu*Z#WVO=|$bP@!5Ch5nF(o;{Sp znHH2k)>0PXv(VOV;e3D|0u34CusoI~ayl+%)fbFOIMRO~%+s(yfEeb_DESW*et>p7 zK)nyp&c6gg%X`)XgB5Sx(^Hwd=jm}gJymJlGgRRko)M<2I@3UaRyh}~O}eNa>2Gjw TTYvT|O&ovj5j`6(!!!Q@&>^q9 literal 0 HcmV?d00001 diff --git a/vwts_scraper/__pycache__/paginate.cpython-312.pyc b/vwts_scraper/__pycache__/paginate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f39559e07a39131c05470b6cd7efc2eb5083271 GIT binary patch literal 4039 zcmai1U2qfE6~6l;$+F}hAwWPOSqMo`0#*!!kkoO;7(&QUW?GXpxM?+ucQJBg$+NpM zKO<+1p+v$ivPl@!G7U~2S~}#xArO>=pGor4PG45zbSieznGDSfZwioUhNph#N-~Bx z(>vO8f6l$($ zTSP6zrGB|v7MRF%v5!lu38>V%`-d;k^I?|%fEaE_Jpm#1yY{u>uzk_KfbUa$SL}j)QL#UwlSZ37h+_W< zyPdY*xBp zuo6)RRZBI@NPlGATKrl!Jl)ba&>L>4zezW~BP=)#n{+j0#rCA)UCGpdBb!9shHj;d zL{Ify`0LbxO89Pngwazh&H83@`4ootuf64vkQPkgfN0SgUd~nPAu%oVxvHI6l_u5J zZf3te#@LXQmij!iCRCdmEaaS6=?ux3Lz+EM`g{)zpgXg_%W9}``x>(I--25EUag6>^E_A!%*+0-OxF)rjg84zXZWnewO!IPdvq~P z>|&2FYhjJIk{Ob~cCoR)>|^_-U2LBa_M~3N#o6yw9HKZz^#hL3qSz<7aTezC2QpX#G}v1SK3>0U$M6P_iA4^;M8vvx)MS}vALIz(=R)fb>!u9%y( zrv1k7pMPZg-@%s~qjJEBf7}2;m5-Bh#xn>|i~A$5pRuPlm${PQ9te^&TslOz4he>XzfK z3^-CRIzyVhJ#N4T@Iv_a4ZR`qlkmk0H~>QbHYc zk->eI88v$h zbwD>FP(GGWExl#nfaB^(rV^I16jCUdm{&oDS$Qe*HRU4NNPf+9{i@;4>%PWPQ}Z{z z#?hyTHs&GS=v`w*1KCa**%ucv$n2TTJi9kQUN_|M$gn)wMjFSHm_I&J;}=WX7YA zF$W}6WI&31j$Rby$3USL+yw?em}3_n8fSo_78(z@Wn4i>&cYmG(mE5U)0dGm;p~P!(!|td?xlCNLVr+hDL@00^7F?UG0zEA%+Y7q^p|a zGcDChnbDr4rW*ugu*!JS)E)1Z{XP1C6-y?2S2=g&Q#eLnwk(f3%X zIdsZ5;mdcP4V6|uRhVCvSYYXGCInu{bbRe^$ZgGT9ot6sJ(L}?>z021m2W=G&lSAh zG!N$I3RcW-q+?#k0`}PgZ^v@!GerQ!A#pKKKMFH+JU|p&oi?{UHsW%Sm>~IlV+oFm zG@}dFX<@Wd(qfvO227@f)9jDZ48$YN(4;qDGnEw@5|<$t5(_!);-e929fSx?;8Z?{ zxkK(bq&bc_jSlc5`ncyeTQ5LG3}luTNsO2GuAqn2WT$RmQoVPmd{ns0xOrf zMb}br28(uKROk_Ru`V`a)t=OEu;d3;4g0|^+DWw#dT!9es5^x4?z;^%~B8vqWRX`nGj;iaQS`W9Hg5N+h z`$}csk>UihSmeu7e9y5;dT@Dl1=1c!*GXBPXt%% z6yqvVHco#L&?Vk)xdZb|{Rg-4Aug!;mw^t>U5KFMpo>FWY#i5plU_8n0t9UP4Cc@J zQ-P!gaf5^)X@5>h&8KOUWz4>LTuxZ#e&|T?WUu4dry7Y^qSts7v%_^p7?tq^k;gzN zI#3{R#CXyX_r?H%F%1pFq<}{%QmJ^lZ%{X4dk#dAnOVkquwTGu{v8C4?q_vPA;H_cgYtR?>yF43NFd)D7)Fb;P}?e zri@r>Y#v{GthW?gkQXPO$^U%fr<1M4c`GxWv&D24p0*$Fm=0{pY$`Pc#=DLkC@ol& zZ=Lw%g!uJ>MdM<*fnj*Jh*WHeXk?!HrE@3UXB-GaXUtJ!>Nl#6KcPX$!owd==pdSfDFbm_>=K# z{VTf-4&#@8x2~-}sioq2yFs^2K5XhBP*)JX6=gyGub&CQf3sy@yFOD!W$*E5{qk4AtZ#x2gK}WuaaO=FKugRS4dus=Q>W5XyIz8LG4Ecva}ULp`G% GJN*wQ(qyFo literal 0 HcmV?d00001 diff --git a/vwts_scraper/__pycache__/parse.cpython-312.pyc b/vwts_scraper/__pycache__/parse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..712c0284c02e2347131354c0168438267387ed3d GIT binary patch literal 2810 zcmb7GYit`u5Z?9O`FW0$xCxJj;E+Iaw6Uog9xaa+C>5n?5CV!6+J>v|;@I+wy*r@h zECqy+Es7kgAQw^8ls^O&iJ%rWyvkoB{+uxn_XH9W;s^gs0}|vHvzI%!4MD(4x3e=h zJ2SgG^X+~a4EhnY>%#*n#f{KUlyMqs50=MaP(u=$L=u)SSSCHS;4Fln4L7VES17mOw2GviF;!N>NEZ;#YpG(I#w zt-rE4Vtj6#ufI@#k&Z3t8DlB>3yo<)z-I71S=7|zw3dD}r{zsnp}m{Psz$PLOttLh z1Ioa+{1ixQD1nMtN2$v-Y()v5wxzq!!5TtEm+nf@nQNNla`cLf&M5fkRVwGGq+k!% zHVL#+G>SD%VkPdhYgOpbBCE4tai@+`fi>#DszFDlv#F3XzC*`BEhe114G6sIJ_k|+4IK7_K8L)2OXT} ztg#3OzEpp?{z}AR&KYOn&_6L2qlFGB`Ak$%i56G)sB)&zdnG@rCe^eY5evbXn#(8S zv5^DPCLw0BGen*?(J%A>3w<-Hnpd{RV$VFQDp8`vrgKEg*b`Q8Okhk`QZhNqmdR)& zE%;5hBB$lJYWi2{R4yxY;yfvD#maCBKz>Wt#dpBeeM<3UGlE)8s>y ziSmT8achGgE8z>F&L+R%d%mNxr@Y7Ly=eiz&%MhnOfMd8jBIQ0+Z~OjH(24z{P*2G zb7Cz~O&k}GrH;j#J-xO4)%~>t)dR;4HG8%!4xAhQV0dZZ%*~5^OOF`g9lv_mUjOfi z^#xiXaNoKb@m;gy%Mc>Eq87a%uxjy-|5J+qO8>1ENjKzsyB4YU)h03Tu}-dLvBW`@ ztzX&g`YCcc2iCY0-Rm_?@7+5KF4R#qkG7;x+g`~zoxBM=T0qJ*1i8= z-QTwEgBbj}k8IO@>*89#(F9!UhP@2vfm9nwE`{TGg=*da)$Ch$CwXfxZPWs3IBb-xrm(cX8k`m>D@9okm#3g~acJD@@Ykm4+W!g=GPbuhxd5~78WNYrFJEk`r5 zqKFB(5Skft#-mv=BR?i!aupa~!8-~*HJOp4+Nh+OUP)AC$`X14g#j(yo+YYKxQF2; z$z?}#`Fu`+(gl18Q{1t=jK*_WRnDq~TR`}vq8>@hH{U&+1%wzrxH+Qa#q8gNlB|l! zw6cQz0t=|2sWUlZvNQ^l$!ZzW3tGpqFHhyr5-zS%btm66dt4&4A~v&qiTUwhyf?6 zLSi7f20n$BFCdJcpS{10Fq<2p(Tga^J&XZjI>WV&YKL+CEsf4CCGJ9OY_6x)U+tfl z=K8S9zWeLY&;lv3ZJfHN!S4W-$UsS~OqZuC z>2msLcAkKn?w(q-8Z`!PYjodU@-#zTCH6v~yV=uMJ5)Vn4DE09JaG}>VE-I$c6QYQ z)xhh)76bf`;cLGO57xtj^Sc&VBRtp$-&)$$^ad*avj6z>g0!^t3$`BK_l_em<)_nzPQADv&s{n%<`uEpE1;T)Wu>N?ub z!u%rKsBsszM$aO_rQHD2jkn>hgr`sho=xW>^%7jBpBocm%pod&z}Pap~l+OG}V!3ti}Gc_gU;!{(EK0r+7urfyICYo$I z33nyE$E;MdJiUvoUAMwm@!plmNm^RIhuj8A^hQ@6f)36U;|pl}PssNJx~}C8;jX!2 p3jtZ!uuMs7s0)YYGA#sTfnBDgHR8he;<dZKWbi7R; zv_?e2luC6AEM|+euGk-G7q+3#{9I}O%+zUPMyWt-q!qt5Ti8GR*xhr-o;aj*(W)bP z?(>}c-E+=8_Z$D!YBgc-{EKJ|WC#rV8)_(zx=^`)6)Nvz6z0Y#oKgjGH!hbdw~A6z znxHzQacgi4>e`?-q;u;+dbeI~>w<=m(QOQw+$OoL51K<3wi#`%MKM_yC`#LGUF1{g72vaa;T6~+7tDgzig&V>b!W4P; z?RQD>4QPBRo{<~EH^SGlNWKXxd@Y_T$lpLZgc}|A5Uhju5IQF=>Guxq7#Mv1kW(e8 zeKaTOS(;`2kubEi!;x5+lgwi@;~zQc33^%1sg^8JFT>IvE)w+*vywUAibhzDl}v>p zua5S{(^=wT+%9IIL29*#`MX%A8_J8I{?fqg3e+IFwqkq)f$`ygE=` z98qeNS(y>n@EWFu*90n*v0@oO8(8sNE40HnkH*K+kr>srAt zaqY65KF+olUIR9?^4bg7C5@sN&R0oX$2BV2C=;)xOykhJrposXU`f?cKor;Wda%C@ z!5_mIHIkbw;q{d1IGjjE#baR|kHHFdUJom%T!tsL5f?yv=f|HDLcop$=@zJ4<$>wZ9a@^Uy>;Hm&WanM6+#WG#}jz zc60EKLfqPTJLmeT9y!YFlayi5<=65ycsKKRPb_}lp58p ztQXj#^c730LmB1qakz;BTZ>~#jdCe-lyU$&x(8ya>Z}iP zaw`P3|8$J%M)(Tv4R9OwJd&fH{(tO#vcb7A_8F`poJT|laU}7KP~=F!i-gaC*W3_p z0_C}Z7*2=DM8dO!@v(h!4YigL4WLy(jzC*p(~Hj!tzc7aBoL%2;Tv(HV@V$A80EMq z+vjqPy~42_Ow2VBVPYZInAhhzMu*4!;mFuYx}6&H`@9?-^t)u)hq&wBf8deKd7k8< zB4Ij)F44uy@4ZC|mjN-*I~mWPVNqakePj&Lp8}t2BY{hC9?3M~57Hign4(We>dqc& zT-Om`BjI2SU5|G!O_IVD5WgwDfxzED8fE(TCbEj~6@xBfNj2IPLmX)F^7%7l4EQrf z@+*jCAIYG+ju@PeW@WVFT2yb3p%Va-Z;^)|;z#cxg)wk528OwN_U1pP&yr*e9euE_ z_-3AZil_3XzAjFPXMw}sop_HFK9ju&o)Q!J{l?IBg|vK_mbDaYg0!4Q*n!^W$!|vt z(G0TXXBX%rAL330l<@n!F(s9s8rOjS!{h3sWYf5+kCe2Wp9|6l?>!6pBsGxF@l}s^ zuIx0|xXvY;U|a`k;E5r!CT|S^1K)j0w!{$WQh++!^4|NCFx^Lv_doRu9@$}cFPkENh`w{4 z)`|&?6rY$u=AbgH(+3!zWEoXOEPk>z!TFC^SAsko@5=KO$9 z?#qwoGebe6Hva*o;o#E=rbf~HXt#uquyTB%=v(grj_65A4JC5`Fe1aTJHgEp*kTD*)-Xq`6^XU!;VjWQM|6rr z=M<48HclNAi7gXD4|JHRBoWLSYbSQ*^u|Q{xB9AigZZrCv_T*`{>V;sPldz{16kLg z(6M_F!}s9Tx$4@~kXY@^GznE5nUO^`?&y7h;l@o544B21cr|OTofx=fwjnjf(?-Fu zan{gV(3vcqG~XdA-`{p_Te>Gpv`h@$>g-SU&k<`xV$B@UDiW=;MB7Z;#L$AHGNn1c zeq#TEqb_wkv-L}T*0FtJ|GY;3&feGeCSu7W$%De`jZ?>`jtbUYvzmcL9adR2@ytRA zk=&7dHT~SV*Rmz+?qgfD4GC4wY)v|nLt<&8Xl~5aHzc0PS;|15o=kUr_|oa)nZua_ z)8*65G$pj|N~jgEy{Xlyvea{_S5u)(mr&Y1Yv@=gt4yiS?@btQ*{X7u+SJRUg-kqi z$5x+i{MObqZztY2oHL}hre719Hwl|Yg=+t-Js{c3=PkCg`%mu|YPQX&ra3U=0KO@= zX}8e3XXa(FVD&y6OlaMY-?5bCYHAa<8&?#WHUS!j~7XXS-+BR!w2gg-3 zq{h-OXUm=Uu{vYZ>A}Q~1e0@=C0{t#lGrtGb0jyMIg#p4g~Zx6(bkr;S1kaDy{Gpk zxzvu-hLrcJI=%ZNb1I(MHKoqPrgly-)9XHYb!NlNPGQp?p?9xP{`9P6-+c9|)bnZW z2fw*cpV)VcsJqkAIaT@b@uVvGlxT0f+LRvts717|pSPDKkEFY2?agSRnufF~TeCi~ zKUcHrVjxqKt?x|l)YJ)N|Ma2h!I_G`sApnA{ef)F!Q?KfvL;tmceOTiDBIX2R&{-#U9^Il2X@SA zm$x@|aX7v9LLhY{(=c5-Z3ScR##iT7H>WGW(iKi*=+*^GMUqb+f$)w^Q!_hfEqijU zZJC!oUZ2>Rh>MmwP|C?Vv})GS zG;gpZj-R{s} z`ykNwBc{?0;R$>pe|nRfQ#&s2{$O`%bGke2{lmzWk&i~_ntR3O-YHIO-a0)XHgBIH zgtdb+F9@akzBBA!)FOHCkb?X>O~rjkC782X(tCh040}8be05`z9xewTq;W|HX>!Ex zV<6W+jvt}9;Zgaf$R~XRq)LAn(tGhXkbrW)e@k-y&u4j*@6n}!rlFfhz6B&*B*sN! zT>cI)!oOBSup-S!29F0NYXGta&Fw@ll=42tn1iT31h0pN{!oO91?gWgZfK(am)Py_ z0;}TqE$p?wW5f?w?QN{#Hr8?*+bm+6Z)0^oVD)!&l@q=>U8Sh2Oc}De#tHSJ?ih|& MCJ#NpP${qQU&Y6YcK`qY literal 0 HcmV?d00001 diff --git a/vwts_scraper/__pycache__/state.cpython-312.pyc b/vwts_scraper/__pycache__/state.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a18192f512158cf081d59178948e8b153548bb2 GIT binary patch literal 1449 zcmah}&1)M+6rb6hm1IedNerc}>r_63dJ(pu1nNU+DY$h4uAAC21rb;_yJLGDd8N*b zVp*slmq416b&Csy8hS~v4<T%x`TTNIpro$dhw}mMsiF-O5ZZ?Yu~&# z^LuaikN*B70(y6FWci_i&{Jic5a}tMtDxLLJ}Mv|`$iELFi5j#te6E8BLh*>C)>Ci zT_C(nVl0vL!$H7j3ozQk4Rdy2WzBR*6Rr0Vq`YGCPHKN_mILr4^bBT}q3&uyT#^-1PM4WAm4D`ik6R zZke)NH*a}OrW~z03qg^xGmxVX7Jm;|105yKHLtc3BaKfEiFIpgeX2RRb+Iw^n2a8# z(%;ovsiBArbs!YIgpWFr#-|Ai;PV2zIy#w#|H5cnB?Dp?gvz36Id-C*Nn*dL=&y7nwrOCNd>T<-mhUD1}re%kCmQ z(G@;U5c;&*3v#?tB^0)$L6b4vGA@alJ!ut7uFqv3qpt4=S`{z)w9xAl>xY7ps5mwZ z>HsYMF))StU~~FFqaIJ(s;*aW){X`SH)^+QnpSwgiT%GGsIPv<-0sb~Pj;-+>0U#2 zZ056P8dpQ6X~-*#IxhEuK*kh~pH0&HOn{-i3uChQ-jTn0i{L=C(D`nn8kDUkC z%lL8k24xq({f2rq1`O~c1I_dS8UB-uw#evFBE6Bkoov3p_37@&ADO ze#Ek{Nev?Z3Rv6382^O|Pf_Y`^yZWPv(4;5|M^C;9W(Jen_e4%*z!9{w9^FVag(=^ XlAEHVW&4bQFKMrmTjV)XQWyRg21IA= literal 0 HcmV?d00001 diff --git a/vwts_scraper/config.py b/vwts_scraper/config.py index 27ec007..5df7f32 100644 --- a/vwts_scraper/config.py +++ b/vwts_scraper/config.py @@ -1,10 +1,12 @@ """Константы.""" import socket +from pathlib import Path # Глобальный таймаут сокетов socket.setdefaulttimeout(30) +OUTPUT_DIR = Path("vwts_data") # куда сохраняем данные DELAY = 1.5 # секунд между запросами TIMEOUT = (10, 30) # (connect, read) TOPICS_PER_FILE = 100 diff --git a/vwts_scraper/output.py b/vwts_scraper/output.py index d94ef7c..64e128b 100644 --- a/vwts_scraper/output.py +++ b/vwts_scraper/output.py @@ -2,10 +2,8 @@ import json, logging from datetime import datetime -from pathlib import Path -from .config import TOPICS_PER_FILE +from .config import TOPICS_PER_FILE, OUTPUT_DIR -OUTPUT_DIR = Path("vwts_data") log = logging.getLogger(__name__) diff --git a/vwts_scraper/run.py b/vwts_scraper/run.py index 208c040..b34e0b6 100644 --- a/vwts_scraper/run.py +++ b/vwts_scraper/run.py @@ -1,7 +1,9 @@ """Основной цикл: обход страниц раздела → темы → посты → сохранение.""" import sys, logging -from .config import HEADERS +from pathlib import Path +from urllib.parse import urlparse +from .config import OUTPUT_DIR from .fetch import get, session from .paginate import count, verify_last from .parse import parse_topics, parse_posts, parse_tags @@ -85,11 +87,13 @@ def run(section_url: str): posts, tags = [], parse_tags(topic_soup) # ── Обход страниц темы ──────────────────────────────────── + base = f"{urlparse(section_url).scheme}://{urlparse(section_url).netloc}" + for tpp in range(1, topic_pages + 1): if tpp == 1: posts += parse_posts(topic_soup) else: - tp2 = get(f"https://vwts.ru/forum/topic/{tid}/page-{tpp}") + tp2 = get(f"{base}/forum/topic/{tid}/page-{tpp}") if tp2: posts += parse_posts(tp2) @@ -106,7 +110,6 @@ def run(section_url: str): log.info("=" * 60) log.info(f"✅ '{section_name}' — {st['topic_count']} тем " f"в {st['file_index']+1} частях") - for f in sorted((__import__('pathlib').Path('vwts_data') / - section_slug).glob("part_*.jsonl")): + for f in sorted((OUTPUT_DIR / section_slug).glob("part_*.jsonl")): log.info(f" 📄 {f.name} ({f.stat().st_size/1024/1024:.1f} MB)") log.info("=" * 60) diff --git a/vwts_scraper/state.py b/vwts_scraper/state.py index 291cb1e..6917c95 100644 --- a/vwts_scraper/state.py +++ b/vwts_scraper/state.py @@ -1,9 +1,7 @@ """Управление состоянием (state.json в папке раздела).""" import json -from pathlib import Path - -OUTPUT_DIR = Path("vwts_data") +from .config import OUTPUT_DIR def _state_file(section_slug: str) -> Path: