From c0b87880ae86bf4d46fec58293a04fd30284f2d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Sat, 5 Sep 2026 08:51:32 +0300 Subject: [PATCH] Implement reusable upload platform --- .gitignore | 3 + README.md | 2 + config.json | 9 +++ site/app.py | 27 ++++++- site/routes/__init__.py | 1 + site/routes/api_bp.py | 26 +++++++ site/static/style.css | 32 +++++++- site/static/vendor/fflate.min.js | 1 + site/templates/index.html | 81 +++++++++++++++++++- upload/README.md | 13 ++++ upload/__init__.py | 1 + upload/backend/__init__.py | 1 + upload/backend/session/__init__.py | 19 +++++ upload/backend/session/add_file.py | 17 ++++ upload/backend/session/cleanup.py | 12 +++ upload/backend/session/create_session.py | 19 +++++ upload/backend/session/get_files.py | 21 +++++ upload/backend/session/state.py | 37 +++++++++ upload/backend/upload_refs/__init__.py | 15 ++++ upload/backend/upload_refs/blueprint.py | 79 +++++++++++++++++++ upload/backend/upload_refs/config.py | 5 ++ upload/backend/upload_refs/pull_file.py | 38 +++++++++ upload/backend/upload_refs/safe_name.py | 13 ++++ upload/config.example.json | 9 +++ upload/frontend/table/add_file_with_dedup.js | 27 +++++++ upload/frontend/table/esc.js | 9 +++ upload/frontend/table/fs.js | 5 ++ upload/frontend/table/init_upload_table.js | 44 +++++++++++ upload/frontend/table/on_files_change.js | 29 +++++++ upload/frontend/table/on_folder_change.js | 37 +++++++++ upload/frontend/table/render.js | 22 ++++++ upload/frontend/table/set_status.js | 5 ++ upload/frontend/upload/put_to_vm.js | 19 +++++ upload/frontend/upload/upload_via_vm.js | 33 ++++++++ upload/frontend/zip/list_zip_files.js | 32 ++++++++ 35 files changed, 734 insertions(+), 9 deletions(-) create mode 100644 config.json create mode 100644 site/routes/__init__.py create mode 100644 site/routes/api_bp.py create mode 100644 site/static/vendor/fflate.min.js create mode 100644 upload/README.md create mode 100644 upload/__init__.py create mode 100644 upload/backend/__init__.py create mode 100644 upload/backend/session/__init__.py create mode 100644 upload/backend/session/add_file.py create mode 100644 upload/backend/session/cleanup.py create mode 100644 upload/backend/session/create_session.py create mode 100644 upload/backend/session/get_files.py create mode 100644 upload/backend/session/state.py create mode 100644 upload/backend/upload_refs/__init__.py create mode 100644 upload/backend/upload_refs/blueprint.py create mode 100644 upload/backend/upload_refs/config.py create mode 100644 upload/backend/upload_refs/pull_file.py create mode 100644 upload/backend/upload_refs/safe_name.py create mode 100644 upload/config.example.json create mode 100644 upload/frontend/table/add_file_with_dedup.js create mode 100644 upload/frontend/table/esc.js create mode 100644 upload/frontend/table/fs.js create mode 100644 upload/frontend/table/init_upload_table.js create mode 100644 upload/frontend/table/on_files_change.js create mode 100644 upload/frontend/table/on_folder_change.js create mode 100644 upload/frontend/table/render.js create mode 100644 upload/frontend/table/set_status.js create mode 100644 upload/frontend/upload/put_to_vm.js create mode 100644 upload/frontend/upload/upload_via_vm.js create mode 100644 upload/frontend/zip/list_zip_files.js diff --git a/.gitignore b/.gitignore index 05fd59c..f8ab551 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,9 @@ htmlcov/ .DS_Store Thumbs.db +# Local frontend tooling +node_modules/ + # Logs / temp *.log /tmp/ diff --git a/README.md b/README.md index 9f74f0c..8889c99 100644 --- a/README.md +++ b/README.md @@ -63,3 +63,5 @@ python site/app.py 5. Слой 3 (обработка файлов) — свой; сюда приходит список файлов сессии после закачки. Подробнее — `PLAN.md`. + +Переиспользуемая инструкция находится в `upload/README.md`. diff --git a/config.json b/config.json new file mode 100644 index 0000000..9dd3def --- /dev/null +++ b/config.json @@ -0,0 +1,9 @@ +{ + "vmUploadUrl": "https://contracts.kube5s.ru/drhider-upload/", + "allowedExt": [".pdf", ".doc", ".docx", ".txt", ".md"], + "maxFileBytes": 52428800, + "maxSessionBytes": 524288000, + "apiPrefix": "/api", + "pullRetries": 3, + "pullRetryDelay": 2 +} \ No newline at end of file diff --git a/site/app.py b/site/app.py index 22caa6b..bf8fe95 100644 --- a/site/app.py +++ b/site/app.py @@ -1,14 +1,25 @@ -from flask import Flask, render_template +import json +import sys +from pathlib import Path + +from flask import Flask, render_template, send_from_directory -VERSION = "0.1.1" +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +with (ROOT / "config.json").open(encoding="utf-8") as config_file: + CONFIG = json.load(config_file) + + +VERSION = "0.1.2" app = Flask(__name__, template_folder="templates", static_folder="static") @app.route("/") def index(): - return render_template("index.html", version=VERSION) + return render_template("index.html", version=VERSION, config=CONFIG) @app.route("/health") @@ -16,5 +27,15 @@ def health(): return "ok", 200 +@app.get("/upload-frontend/") +def upload_frontend(filename): + return send_from_directory(ROOT / "upload" / "frontend", filename) + + +from routes.api_bp import register_api + +register_api(app, CONFIG) + + if __name__ == "__main__": app.run(debug=False, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/site/routes/__init__.py b/site/routes/__init__.py new file mode 100644 index 0000000..5b3e2cb --- /dev/null +++ b/site/routes/__init__.py @@ -0,0 +1 @@ +"""Маршруты демо-обёртки.""" \ No newline at end of file diff --git a/site/routes/api_bp.py b/site/routes/api_bp.py new file mode 100644 index 0000000..4c1e8ae --- /dev/null +++ b/site/routes/api_bp.py @@ -0,0 +1,26 @@ +"""Демо API: переиспользуемый upload_refs и заглушка обработки.""" + +from flask import Blueprint, jsonify, request + +from upload.backend.session import get_files +from upload.backend.upload_refs import create_upload_refs_blueprint + + +def register_api(app, cfg): + app.register_blueprint(create_upload_refs_blueprint(cfg)) + demo = Blueprint("demo_api", __name__, url_prefix=cfg.get("apiPrefix", "/api")) + + @demo.post("/process") + def process(): + data = request.get_json(silent=True) or {} + sid = data.get("session", "") + files = get_files(sid) + if files is None: + return jsonify({"ok": False, "error": "Session not found"}), 404 + return jsonify({ + "ok": True, + "session": sid, + "files": [{"name": name, "size": len(content)} for name, content in files], + }) + + app.register_blueprint(demo) \ No newline at end of file diff --git a/site/static/style.css b/site/static/style.css index 882e513..c4c0d51 100644 --- a/site/static/style.css +++ b/site/static/style.css @@ -1,8 +1,36 @@ :root { color-scheme: light; - font-family: sans-serif; + font-family: Georgia, 'Times New Roman', serif; + color: #17211b; + background: #e9eee8; } body { - margin: 2rem; + margin: 0; + min-height: 100vh; + background: radial-gradient(circle at 10% 0%, #f8fbf4, transparent 38%), #e9eee8; +} + +.page { max-width: 980px; margin: 0 auto; padding: 64px 24px; } +.eyebrow { color: #56745f; font: 700 12px/1.2 sans-serif; letter-spacing: 2px; } +h1 { max-width: 620px; margin: 12px 0; font-size: clamp(38px, 7vw, 72px); line-height: .95; font-weight: 400; } +.lede { max-width: 560px; color: #5d6a61; font: 16px/1.6 sans-serif; } +.toolbar { display: flex; flex-wrap: wrap; gap: 10px; margin: 36px 0 16px; } +button { border: 0; border-radius: 4px; padding: 12px 17px; color: #f7faf4; background: #1f5a3b; font: 700 13px sans-serif; cursor: pointer; } +button.secondary { color: #1f5a3b; background: #cbdcc9; } +button.quiet { color: #5d6a61; background: transparent; } +button:disabled { opacity: .45; cursor: not-allowed; } +.status { min-height: 22px; color: #56745f; font: 13px sans-serif; } +.table-wrap { overflow-x: auto; border-top: 1px solid #b9c7ba; } +table { width: 100%; border-collapse: collapse; font: 14px/1.4 sans-serif; } +th, td { padding: 14px 10px; border-bottom: 1px solid #cbd5cc; text-align: left; } +th { color: #56745f; font-size: 11px; text-transform: uppercase; letter-spacing: 1px; } +td:nth-child(2) { width: 140px; color: #68766c; } +.count { color: #5d6a61; font: 13px sans-serif; } +.result { margin-top: 44px; padding-top: 18px; border-top: 2px solid #1f5a3b; } +.result h2 { font-size: 26px; font-weight: 400; } +.result li { margin: 8px 0; font: 14px sans-serif; } +@media (max-width: 600px) { + .page { padding: 36px 16px; } + .toolbar button { flex: 1 1 42%; } } \ No newline at end of file diff --git a/site/static/vendor/fflate.min.js b/site/static/vendor/fflate.min.js new file mode 100644 index 0000000..df76237 --- /dev/null +++ b/site/static/vendor/fflate.min.js @@ -0,0 +1 @@ +!function(f){typeof module!='undefined'&&typeof exports=='object'?module.exports=f():typeof define!='undefined'&&define.amd?define(f):(typeof self!='undefined'?self:this).fflate=f()}(function(){var _e={};"use strict";_e.deflate=zt,_e.deflateSync=kt,_e.inflate=At,_e.inflateSync=Tt,_e.gzip=It,_e.compress=It,_e.gzipSync=Ut,_e.compressSync=Ut,_e.gunzip=Zt,_e.gunzipSync=qt,_e.zlib=Lt,_e.zlibSync=Bt,_e.unzlib=Nt,_e.unzlibSync=Pt,_e.gzip=It,_e.compress=It,_e.decompress=Jt,_e.decompressSync=Kt,_e.strToU8=nn,_e.strFromU8=rn,_e.zip=dn,_e.zipSync=gn,_e.unzip=zn,_e.unzipSync=kn;var t=(typeof module!='undefined'&&typeof exports=='object'?function(_f){"use strict";var e,r,t,n=";var __w=require('worker_threads');__w.parentPort.on('message',function(m){onmessage({data:m})}),postMessage=function(m,t){__w.parentPort.postMessage(m,t)},close=process.exit;self=global";try{e=require("worker_threads"),r=e.Worker,t=e.isMarkedAsUntransferable}catch(e){}exports.default=r?function(e,o,a,s,u){var i=!1,l=new r(e+n,{eval:!0}).on("error",function(e){return u(e,null)}).on("message",function(e){return u(null,e)}).on("exit",function(e){e&&!i&&u(Error("exited with code "+e),null)});return t&&(s=s.filter(function(e){return!t(e)})),l.postMessage(a,s),l.terminate=function(){return i=!0,r.prototype.terminate.call(l)},l}:function(e,r,t,n,o){setImmediate(function(){return o(Error("async operations unsupported - update to Node 12+ (or Node 10-11 with the --experimental-worker CLI flag)"),null)});var a=function(){};return{terminate:a,postMessage:a}};return _f}:function(_f){"use strict";var e={};_f.default=function(r,t,s,a,n){var o=new Worker(e[t]||(e[t]=URL.createObjectURL(new Blob([r+';addEventListener("error",function(e){e=e.error;postMessage({$e$:[e.message,e.code,e.stack]})})'],{type:"text/javascript"}))));return o.onmessage=function(e){var r=e.data,t=r.$e$;if(t){var s=Error(t[0]);s.code=t[1],s.stack=t[2],n(s,null)}else n(null,r)},o.postMessage(s,a),o};return _f})({}),n=Uint8Array,r=Uint16Array,i=Int32Array,e=new n([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),o=new n([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),s=new n([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),a=function(t,n){for(var e=new r(31),o=0;o<31;++o)e[o]=n+=1<>1|(21845&d)<<1;v[d]=((65280&(g=(61680&(g=(52428&g)>>2|(13107&g)<<2))>>4|(3855&g)<<4))>>8|(255&g)<<8)>>1}var y=function(t,n,i){for(var e=t.length,o=0,s=new r(n);o>h]=f}else for(a=new r(e),o=0;o>15-t[o]);return a},m=new n(288);for(d=0;d<144;++d)m[d]=8;for(d=144;d<256;++d)m[d]=9;for(d=256;d<280;++d)m[d]=7;for(d=280;d<288;++d)m[d]=8;var b=new n(32);for(d=0;d<32;++d)b[d]=5;var w=y(m,9,0),x=y(m,9,1),z=y(b,5,0),k=y(b,5,1),M=function(t){for(var n=t[0],r=1;rn&&(n=t[r]);return n},S=function(t,n,r){var i=n/8|0;return(t[i]|t[i+1]<<8)>>(7&n)&r},A=function(t,n){var r=n/8|0;return(t[r]|t[r+1]<<8|t[r+2]<<16)>>(7&n)},T=function(t){return(t+7)/8|0},D=function(t,r,i){return(null==r||r<0)&&(r=0),(null==i||i>t.length)&&(i=t.length),new n(t.subarray(r,i))};_e.FlateErrorCode={UnexpectedEOF:0,InvalidBlockType:1,InvalidLengthLiteral:2,InvalidDistance:3,StreamFinished:4,NoStreamHandler:5,InvalidHeader:6,NoCallback:7,InvalidUTF8:8,ExtraFieldTooLong:9,InvalidDate:10,FilenameTooLong:11,StreamFinishing:12,InvalidZipData:13,UnknownCompressionMethod:14};var C=["unexpected EOF","invalid block type","invalid length/literal","invalid distance","stream finished","no stream handler",,"no callback","invalid UTF-8 data","extra field too long","date not in range 1980-2099","filename too long","stream finishing","invalid zip data"],I=function(t,n,r){var i=Error(n||C[t]);if(i.code=t,Error.captureStackTrace&&Error.captureStackTrace(i,I),!r)throw i;return i},U=function(t,r,i,a){var u=t.length,f=a?a.length:0;if(!u||r.f&&!r.l)return i||new n(0);var c=!i,p=c||2!=r.i,v=r.i;c&&(i=new n(3*u));var d=function(t){var r=i.length;if(t>r){var e=new n(Math.max(2*r,t));e.set(i),i=e}},g=r.f||0,m=r.p||0,b=r.b||0,w=r.l,z=r.d,C=r.m,U=r.n,F=8*u;do{if(!w){g=S(t,m,1);var E=S(t,m+1,3);if(m+=3,!E){var Z=t[(Y=T(m)+4)-4]|t[Y-3]<<8,q=Y+Z;if(q>u){v&&I(0);break}p&&d(b+Z),i.set(t.subarray(Y,q),b),r.b=b+=Z,r.p=m=8*q,r.f=g;continue}if(1==E)w=x,z=k,C=9,U=5;else if(2==E){var O=S(t,m,31)+257,G=S(t,m+10,15)+4,L=O+S(t,m+5,31)+1;m+=14;for(var B=new n(L),H=new n(19),j=0;j>4)<16)B[j++]=Y;else{var K=0,Q=0;for(16==Y?(Q=3+S(t,m,3),m+=2,K=B[j-1]):17==Y?(Q=3+S(t,m,7),m+=3):18==Y&&(Q=11+S(t,m,127),m+=7);Q--;)B[j++]=K}}var R=B.subarray(0,O),W=B.subarray(O);C=M(R),U=M(W),w=y(R,C,1),z=y(W,U,1)}else I(1);if(m>F){v&&I(0);break}}p&&d(b+131072);for(var X=(1<>4;if((m+=15&K)>F){v&&I(0);break}if(K||I(2),tt<256)i[b++]=tt;else{if(256==tt){_=m,w=null;break}var nt=tt-254;tt>264&&(nt=S(t,m,(1<<(et=e[j=tt-257]))-1)+h[j],m+=et);var rt=z[A(t,m)&$],it=rt>>4;if(rt||I(3),m+=15&rt,W=l[it],it>3){var et=o[it];W+=A(t,m)&(1<F){v&&I(0);break}p&&d(b+131072);var ot=b+nt;if(b>8},E=function(t,n,r){var i=n/8|0;t[i]|=r<<=7&n,t[i+1]|=r>>8,t[i+2]|=r>>16},Z=function(t,i){for(var e=[],o=0;ov&&(v=a[o].s);var d=new r(v+1),g=q(e[l-1],d,0);if(g>i){o=0;var y=0,m=g-i,b=1<i))break;y+=b-(1<>=m;y>0;){var x=a[o].s;d[x]=0&&y;--o){var z=a[o].s;d[z]==i&&(--d[z],++y)}g=i}return{t:new n(d),l:g}},q=function(t,n,r){return-1==t.s?Math.max(q(t.l,n,r+1),q(t.r,n,r+1)):n[t.s]=r},O=function(t){for(var n=t.length;n&&!t[--n];);for(var i=new r(++n),e=0,o=t[0],s=1,a=function(t){i[e++]=t},u=1;u<=n;++u)if(t[u]==o&&u!=n)++s;else{if(!o&&s>2){for(;s>138;s-=138)a(32754);s>2&&(a(s>10?s-11<<5|28690:s-3<<5|12305),s=0)}else if(s>3){for(a(o),--s;s>6;s-=6)a(8304);s>2&&(a(s-3<<5|8208),s=0)}for(;s--;)a(o);s=1,o=t[u]}return{c:i.subarray(0,e),n:n}},G=function(t,n){for(var r=0,i=0;i>8,t[e+2]=255^t[e],t[e+3]=255^t[e+1];for(var o=0;o4&&!j[s[P-1]];--P);var V,Y,J,K,Q=p+5<<3,R=G(u,m)+G(h,b)+f,W=G(u,g)+G(h,M)+f+14+3*P+G(q,j)+2*q[16]+3*q[17]+7*q[18];if(l>=0&&Q<=R&&Q<=W)return L(n,v,t.subarray(l,l+p));if(F(n,v,1+(W15&&(F(n,v,tt[B]>>5&127),v+=tt[B]>>12)}}else V=w,Y=m,J=z,K=b;for(B=0;B255){var rt;E(n,v,V[257+(rt=nt>>18&31)]),v+=Y[rt+257],rt>7&&(F(n,v,nt>>23&31),v+=e[rt]);var it=31&nt;E(n,v,J[it]),v+=K[it],it>3&&(E(n,v,nt>>5&8191),v+=o[it])}else E(n,v,V[nt]),v+=Y[nt]}return E(n,v,V[256]),v+Y[256]},H=new i([65540,131080,131088,131104,262176,1048704,1048832,2114560,2117632]),j=new n(0),N=function(t,s,a,u,h,c){var l=c.z||t.length,v=new n(u+l+5*(1+Math.ceil(l/7e3))+h),d=v.subarray(u,v.length-h),g=c.l,y=7&(c.r||0);if(s){y&&(d[0]=c.r>>3);for(var m=H[s-1],b=m>>13,w=8191&m,x=(1<7e3||q>24576)&&(V>423||!g)){y=B(t,d,0,C,I,U,E,q,G,Z-G,y),q=F=E=0,G=Z;for(var Y=0;Y<286;++Y)I[Y]=0;for(Y=0;Y<30;++Y)U[Y]=0}var J=2,K=0,Q=w,R=N-P&32767;if(V>2&&j==A(Z-R))for(var W=Math.min(b,V)-1,X=Math.min(32767,Z),$=Math.min(258,V);R<=X&&--Q&&N!=P;){if(t[Z+J]==t[Z+J-R]){for(var _=0;_<$&&t[Z+_]==t[Z+_-R];++_);if(_>J){if(J=_,K=R,_>W)break;var tt=Math.min(R,_-2),nt=0;for(Y=0;Ynt&&(nt=it,P=rt)}}}R+=(N=P)-(P=z[N])&32767}if(K){C[q++]=268435456|f[J]<<18|p[K];var et=31&f[J],ot=31&p[K];E+=e[et]+o[ot],++I[257+et],++U[ot],O=Z+J,++F}else C[q++]=t[Z],++I[t[Z]]}}for(Z=Math.max(Z,O);Z=l&&(d[y/8|0]=g,st=l),y=L(d,y+1,t.subarray(Z,st))}c.i=l}return D(v,0,u+T(y)+h)},P=function(){for(var t=new Int32Array(256),n=0;n<256;++n){for(var r=n,i=9;--i;)r=(1&r&&-306674912)^r>>>1;t[n]=r}return t}(),V=function(){var t=-1;return{p:function(n){for(var r=t,i=0;i>>8;t=r},d:function(){return~t}}},Y=function(){var t=1,n=0;return{p:function(r){for(var i=t,e=n,o=0|r.length,s=0;s!=o;){for(var a=Math.min(s+2655,o);s>16),e=(65535&e)+15*(e>>16)}t=i,n=e},d:function(){return(255&(t%=65521))<<24|(65280&t)<<8|(255&(n%=65521))<<8|n>>8}}},J=function(t,r,i,e,o){if(!o&&(o={l:1},r.dictionary)){var s=r.dictionary.subarray(-32768),a=new n(s.length+t.length);a.set(s),a.set(t,s.length),t=a,o.w=s.length}return N(t,null==r.level?6:r.level,null==r.mem?o.l?Math.ceil(1.5*Math.max(8,Math.min(13,Math.log(t.length)))):20:12+r.mem,i,e,o)},K=function(t,n){var r={};for(var i in t)r[i]=t[i];for(var i in n)r[i]=n[i];return r},Q=function(t,n,r){for(var i=t(),e=""+t,o=e.slice(e.indexOf("[")+1,e.lastIndexOf("]")).replace(/\s+/g,"").split(","),s=0;s>>0},ct=function(t,n){return ft(t,n)+4294967296*ft(t,n+4)},lt=function(t,n,r){for(;r;++n)t[n]=r,r>>>=8},pt=function(t,n){var r=n.filename;if(t[0]=31,t[1]=139,t[2]=8,t[8]=n.level<2?4:9==n.level?2:0,t[9]=3,0!=n.mtime&<(t,4,Math.floor(new Date(n.mtime||Date.now())/1e3)),r){t[3]=8;for(var i=0;i<=r.length;++i)t[i+10]=r.charCodeAt(i)}},vt=function(t){31==t[0]&&139==t[1]&&8==t[2]||I(6,"invalid gzip data");var n=t[3],r=10;4&n&&(r+=2+(t[10]|t[11]<<8));for(var i=(n>>3&1)+(n>>4&1);i>0;i-=!t[r++]);return r+(2&n)},dt=function(t){var n=t.length;return(t[n-4]|t[n-3]<<8|t[n-2]<<16|t[n-1]<<24)>>>0},gt=function(t){return 10+(t.filename?t.filename.length+1:0)},yt=function(t,n){var r=n.level,i=0==r?0:r<6?1:9==r?3:2;if(t[0]=120,t[1]=i<<6|(n.dictionary&&32),t[1]|=31-(t[0]<<8|t[1])%31,n.dictionary){var e=Y();e.p(n.dictionary),lt(t,2,e.d())}},mt=function(t,n){return(8!=(15&t[0])||t[0]>>4>7||(t[0]<<8|t[1])%31)&&I(6,"invalid zlib data"),(t[1]>>5&1)==+!n&&I(6,"invalid zlib data: "+(32&t[1]?"need":"unexpected")+" dictionary"),2+(t[1]>>3&4)};function bt(t,n){return"function"==typeof t&&(n=t,t={}),this.ondata=n,t}var wt=function(){function t(t,r){if("function"==typeof t&&(r=t,t={}),this.ondata=r,this.o=t||{},this.s={l:0,i:32768,w:32768,z:32768},this.b=new n(98304),this.o.dictionary){var i=this.o.dictionary.subarray(-32768);this.b.set(i,32768-i.length),this.s.i=32768-i.length}}return t.prototype.p=function(t,n){this.ondata(J(t,this.o,0,0,this.s),n)},t.prototype.push=function(t,r){this.ondata||I(5),this.s.l&&I(4);var i=t.length+this.s.z;if(i>this.b.length){if(i>2*this.b.length-32768){var e=new n(-32768&i);e.set(this.b.subarray(0,this.s.z)),this.b=e}var o=this.b.length-this.s.z;this.b.set(t.subarray(0,o),this.s.z),this.s.z=this.b.length,this.p(this.b,!1),this.b.set(this.b.subarray(-32768)),this.b.set(t.subarray(o),32768),this.s.z=t.length-o+32768,this.s.i=32766,this.s.w=32768}else this.b.set(t,this.s.z),this.s.z+=t.length;this.s.l=1&r,(this.s.z>this.s.w+8191||r)&&(this.p(this.b,r||!1),this.s.w=this.s.i,this.s.i-=2),r&&(this.s=this.o={},this.b=j)},t.prototype.flush=function(t){if(this.ondata||I(5),this.s.l&&I(4),this.p(this.b,!1),this.s.w=this.s.i,this.s.i-=2,t){var r=new n(6);r[0]=this.s.r>>3;var i=L(r,this.s.r,j);this.s.r=0,this.ondata(r.subarray(0,i>>3),!1)}},t}();_e.Deflate=wt;var xt=function(){return function(t,n){ut([_,function(){return[at,wt]}],this,bt.call(this,t,n),function(t){var n=new wt(t.data);onmessage=at(n)},6,1)}}();function zt(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),st(t,n,[_],function(t){return et(kt(t.data[0],t.data[1]))},0,r)}function kt(t,n){return J(t,n||{},0,0)}_e.AsyncDeflate=xt;var Mt=function(){function t(t,r){"function"==typeof t&&(r=t,t={}),this.ondata=r;var i=t&&t.dictionary&&t.dictionary.subarray(-32768);this.s={i:0,b:i?i.length:0},this.o=new n(32768),this.p=new n(0),i&&this.o.set(i)}return t.prototype.e=function(t){if(this.ondata||I(5),this.d&&I(4),this.p.length){if(t.length){var r=new n(this.p.length+t.length);r.set(this.p),r.set(t,this.p.length),this.p=r}}else this.p=t},t.prototype.c=function(t){this.s.i=+(this.d=t||!1);var n=this.s.b,r=U(this.p,this.s,this.o);this.ondata(D(r,n,this.s.b),this.d),this.o=D(r,this.s.b-32768),this.s.b=this.o.length,this.p=D(this.p,this.s.p/8|0),this.s.p&=7},t.prototype.push=function(t,n){this.e(t),this.c(n)},t}();_e.Inflate=Mt;var St=function(){return function(t,n){ut([$,function(){return[at,Mt]}],this,bt.call(this,t,n),function(t){var n=new Mt(t.data);onmessage=at(n)},7,0)}}();function At(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),st(t,n,[$],function(t){return et(Tt(t.data[0],ot(t.data[1])))},1,r)}function Tt(t,n){return U(t,{i:2},n&&n.out,n&&n.dictionary)}_e.AsyncInflate=St;var Dt=function(){function t(t,n){this.c=V(),this.l=0,this.v=1,wt.call(this,t,n)}return t.prototype.push=function(t,n){this.c.p(t),this.l+=t.length,wt.prototype.push.call(this,t,n)},t.prototype.p=function(t,n){var r=J(t,this.o,this.v&>(this.o),n&&8,this.s);this.v&&(pt(r,this.o),this.v=0),n&&(lt(r,r.length-8,this.c.d()),lt(r,r.length-4,this.l)),this.ondata(r,n)},t.prototype.flush=function(t){wt.prototype.flush.call(this,t)},t}();_e.Gzip=Dt,_e.Compress=Dt;var Ct=function(){return function(t,n){ut([_,tt,function(){return[at,wt,Dt]}],this,bt.call(this,t,n),function(t){var n=new Dt(t.data);onmessage=at(n)},8,1)}}();function It(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),st(t,n,[_,tt,function(){return[Ut]}],function(t){return et(Ut(t.data[0],t.data[1]))},2,r)}function Ut(t,n){n||(n={});var r=V(),i=t.length;r.p(t);var e=J(t,n,gt(n),8),o=e.length;return pt(e,n),lt(e,o-8,r.d()),lt(e,o-4,i),e}_e.AsyncGzip=Ct,_e.AsyncCompress=Ct;var Ft=function(){function t(t,n){this.v=1,this.r=0,Mt.call(this,t,n)}return t.prototype.push=function(t,r){if(Mt.prototype.e.call(this,t),this.r+=t.length,this.v){var i=this.p.subarray(this.v-1),e=i.length>3?vt(i):4;if(e>i.length){if(!r)return}else this.v>1&&this.onmember&&this.onmember(this.r-i.length);this.p=i.subarray(e),this.v=0}Mt.prototype.c.call(this,0),this.s.f&&!this.s.l?(this.v=T(this.s.p)+9,this.s={i:0},this.o=new n(0),this.push(new n(0),r)):r&&Mt.prototype.c.call(this,r)},t}();_e.Gunzip=Ft;var Et=function(){return function(t,n){var r=this;ut([$,nt,function(){return[at,Mt,Ft]}],this,bt.call(this,t,n),function(t){var n=new Ft(t.data);n.onmember=function(t){return postMessage(t)},onmessage=at(n)},9,0,function(t){return r.onmember&&r.onmember(t)})}}();function Zt(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),st(t,n,[$,nt,function(){return[qt]}],function(t){return et(qt(t.data[0],t.data[1]))},3,r)}function qt(t,r){var i=vt(t);return i+8>t.length&&I(6,"invalid gzip data"),U(t.subarray(i,-8),{i:2},r&&r.out||new n(dt(t)),r&&r.dictionary)}_e.AsyncGunzip=Et;var Ot=function(){function t(t,n){this.c=Y(),this.v=1,wt.call(this,t,n)}return t.prototype.push=function(t,n){this.c.p(t),wt.prototype.push.call(this,t,n)},t.prototype.p=function(t,n){var r=J(t,this.o,this.v&&(this.o.dictionary?6:2),n&&4,this.s);this.v&&(yt(r,this.o),this.v=0),n&<(r,r.length-4,this.c.d()),this.ondata(r,n)},t.prototype.flush=function(t){wt.prototype.flush.call(this,t)},t}();_e.Zlib=Ot;var Gt=function(){return function(t,n){ut([_,rt,function(){return[at,wt,Ot]}],this,bt.call(this,t,n),function(t){var n=new Ot(t.data);onmessage=at(n)},10,1)}}();function Lt(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),st(t,n,[_,rt,function(){return[Bt]}],function(t){return et(Bt(t.data[0],t.data[1]))},4,r)}function Bt(t,n){n||(n={});var r=Y();r.p(t);var i=J(t,n,n.dictionary?6:2,4);return yt(i,n),lt(i,i.length-4,r.d()),i}_e.AsyncZlib=Gt;var Ht=function(){function t(t,n){Mt.call(this,t,n),this.v=t&&t.dictionary?2:1}return t.prototype.push=function(t,n){if(Mt.prototype.e.call(this,t),this.v){if(this.p.length<6&&!n)return;this.p=this.p.subarray(mt(this.p,this.v-1)),this.v=0}n&&(this.p.length<4&&I(6,"invalid zlib data"),this.p=this.p.subarray(0,-4)),Mt.prototype.c.call(this,n)},t}();_e.Unzlib=Ht;var jt=function(){return function(t,n){ut([$,it,function(){return[at,Mt,Ht]}],this,bt.call(this,t,n),function(t){var n=new Ht(t.data);onmessage=at(n)},11,0)}}();function Nt(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),st(t,n,[$,it,function(){return[Pt]}],function(t){return et(Pt(t.data[0],ot(t.data[1])))},5,r)}function Pt(t,n){return U(t.subarray(mt(t,n&&n.dictionary),-4),{i:2},n&&n.out,n&&n.dictionary)}_e.AsyncUnzlib=jt;var Vt=function(){function t(t,n){this.o=bt.call(this,t,n)||{},this.G=Ft,this.I=Mt,this.Z=Ht}return t.prototype.i=function(){var t=this;this.s.ondata=function(n,r){t.ondata(n,r)}},t.prototype.push=function(t,r){if(this.ondata||I(5),this.s)this.s.push(t,r);else{if(this.p&&this.p.length){var i=new n(this.p.length+t.length);i.set(this.p),i.set(t,this.p.length)}else this.p=t;this.p.length>2&&(this.s=31==this.p[0]&&139==this.p[1]&&8==this.p[2]?new this.G(this.o):8!=(15&this.p[0])||this.p[0]>>4>7||(this.p[0]<<8|this.p[1])%31?new this.I(this.o):new this.Z(this.o),this.i(),this.s.push(this.p,r),this.p=null)}},t}();_e.Decompress=Vt;var Yt=function(){function t(t,n){Vt.call(this,t,n),this.queuedSize=0,this.G=Et,this.I=St,this.Z=jt}return t.prototype.i=function(){var t=this;this.s.ondata=function(n,r,i){t.ondata(n,r,i)},this.s.ondrain=function(n){t.queuedSize-=n,t.ondrain&&t.ondrain(n)}},t.prototype.push=function(t,n){this.queuedSize+=t.length,Vt.prototype.push.call(this,t,n)},t}();function Jt(t,n,r){return r||(r=n,n={}),"function"!=typeof r&&I(7),31==t[0]&&139==t[1]&&8==t[2]?Zt(t,n,r):8!=(15&t[0])||t[0]>>4>7||(t[0]<<8|t[1])%31?At(t,n,r):Nt(t,n,r)}function Kt(t,n){return 31==t[0]&&139==t[1]&&8==t[2]?qt(t,n):8!=(15&t[0])||t[0]>>4>7||(t[0]<<8|t[1])%31?Tt(t,n):Pt(t,n)}_e.AsyncDecompress=Yt;var Qt=function(t,r,i,e){for(var o in t){var s=t[o],a=r+o,u=e;Array.isArray(s)&&(u=K(e,s[1]),s=s[0]),ArrayBuffer.isView(s)?i[a]=[s,u]:(i[a+="/"]=[new n(0),u],Qt(s,a,i,e))}},Rt="undefined"!=typeof TextEncoder&&new TextEncoder,Wt="undefined"!=typeof TextDecoder&&new TextDecoder,Xt=0;try{Wt.decode(j,{stream:!0}),Xt=1}catch(t){}var $t=function(t){for(var n="",r=0;;){var i=t[r++],e=(i>127)+(i>223)+(i>239);if(r+e>t.length)return{s:n,r:D(t,r-1)};e?3==e?(i=((15&i)<<18|(63&t[r++])<<12|(63&t[r++])<<6|63&t[r++])-65536,n+=String.fromCharCode(55296|i>>10,56320|1023&i)):n+=String.fromCharCode(1&e?(31&i)<<6|63&t[r++]:(15&i)<<12|(63&t[r++])<<6|63&t[r++]):n+=String.fromCharCode(i)}},_t=function(){function t(t){this.ondata=t,Xt?this.t=new TextDecoder:this.p=j}return t.prototype.push=function(t,r){if(this.ondata||I(5),r=!!r,this.t)return this.ondata(this.t.decode(t,{stream:!0}),r),void(r&&(this.t.decode().length&&I(8),this.t=null));this.p||I(4);var i=new n(this.p.length+t.length);i.set(this.p),i.set(t,this.p.length);var e=$t(i),o=e.s,s=e.r;r?(s.length&&I(8),this.p=null):this.p=s,this.ondata(o,r)},t}();_e.DecodeUTF8=_t;var tn=function(){function t(t){this.ondata=t}return t.prototype.push=function(t,n){this.ondata||I(5),this.d&&I(4),this.ondata(nn(t),this.d=n||!1)},t}();function nn(t,r){if(r){for(var i=new n(t.length),e=0;e>1)),a=0,u=function(t){s[a++]=t};for(e=0;es.length){var h=new n(a+8+(o-e<<1));h.set(s),s=h}var f=t.charCodeAt(e);f<128||r?u(f):f<2048?(u(192|f>>6),u(128|63&f)):f>55295&&f<57344?(u(240|(f=65536+(1047552&f)|1023&t.charCodeAt(++e))>>18),u(128|f>>12&63),u(128|f>>6&63),u(128|63&f)):(u(224|f>>12),u(128|f>>6&63),u(128|63&f))}return D(s,0,a)}function rn(t,n){if(n){for(var r="",i=0;i65535&&I(9),n+=i+4}return n},hn=function(t,n,r,i,e,o,s,a){var u=i.length,h=r.extra,f=a&&a.length,c=un(h);lt(t,n,null!=s?33639248:67324752),n+=4,null!=s&&(t[n++]=20,t[n++]=r.os),t[n]=20,n+=2,t[n++]=r.flag<<1|(o<0&&8),t[n++]=e&&8,t[n++]=255&r.compression,t[n++]=r.compression>>8;var l=new Date(null==r.mtime?Date.now():r.mtime),p=l.getFullYear()-1980;if((p<0||p>119)&&I(10),lt(t,n,p<<25|l.getMonth()+1<<21|l.getDate()<<16|l.getHours()<<11|l.getMinutes()<<5|l.getSeconds()>>1),n+=4,-1!=o&&(lt(t,n,r.crc),lt(t,n+4,o<0?-o-2:o),lt(t,n+8,r.size)),lt(t,n+12,u),lt(t,n+14,c),n+=16,null!=s&&(lt(t,n,f),lt(t,n+6,r.attrs),lt(t,n+10,s),n+=14),t.set(i,n),n+=u,c)for(var v in h){var d=h[v],g=d.length;lt(t,n,+v),lt(t,n+2,g),t.set(d,n+4),n+=4+g}return f&&(t.set(a,n),n+=f),n},fn=function(t,n,r,i,e){lt(t,n,101010256),lt(t,n+8,r),lt(t,n+10,r),lt(t,n+12,i),lt(t,n+16,e)},cn=function(){function t(t){this.filename=t,this.c=V(),this.size=0,this.compression=0}return t.prototype.process=function(t,n){this.ondata(null,t,n)},t.prototype.push=function(t,n){this.ondata||I(5),this.c.p(t),this.size+=t.length,n&&(this.crc=this.c.d()),this.process(t,n||!1)},t}();_e.ZipPassThrough=cn;var ln=function(){function t(t,n){var r=this;n||(n={}),cn.call(this,t),this.d=new wt(n,function(t,n){r.ondata(null,t,n)}),this.compression=8,this.flag=en(n.level)}return t.prototype.process=function(t,n){try{this.d.push(t,n)}catch(t){this.ondata(t,null,n)}},t.prototype.push=function(t,n){cn.prototype.push.call(this,t,n)},t}();_e.ZipDeflate=ln;var pn=function(){function t(t,n){var r=this;n||(n={}),cn.call(this,t),this.d=new xt(n,function(t,n,i){r.ondata(t,n,i)}),this.compression=8,this.flag=en(n.level),this.terminate=this.d.terminate}return t.prototype.process=function(t,n){this.d.push(t,n)},t.prototype.push=function(t,n){cn.prototype.push.call(this,t,n)},t}();_e.AsyncZipDeflate=pn;var vn=function(){function t(t){this.ondata=t,this.u=[],this.d=1}return t.prototype.add=function(t){var r=this;if(this.ondata||I(5),2&this.d)this.ondata(I(4+8*(1&this.d),0,1),null,!1);else{var i=nn(t.filename),e=i.length,o=t.comment,s=o&&nn(o),a=e!=t.filename.length||s&&o.length!=s.length,u=e+un(t.extra)+30;e>65535&&this.ondata(I(11,0,1),null,!1);var h=new n(u);hn(h,0,t,i,a,-1);var f=[h],c=function(){for(var t=0,n=f;t65535&&M(I(11,0,1),null),k)if(g<16e4)try{M(null,kt(i,h))}catch(t){M(t,null)}else c.push(zt(i,h,M));else M(null,i)},g=0;g65535&&I(11);var y=c?kt(h,f):h,m=y.length,b=V();b.p(h),e.push(K(f,{size:h.length,crc:b.d(),c:y,f:M,m:v,u:l!=a.length||v&&p.length!=d,o:o,compression:c})),o+=30+l+g+m,s+=76+2*(l+g)+(d||0)+m}for(var w=new n(s+22),x=o,z=s-o,k=0;k0){var e=Math.min(this.c,t.length),o=t.subarray(0,e);if(this.c-=e,this.d?this.d.push(o,!this.c):this.k[0].push(o),(t=t.subarray(e)).length)return this.push(t,r)}else{var s=0,a=0,u=void 0,h=void 0;this.p.length?t.length?((h=new n(this.p.length+t.length)).set(this.p),h.set(t,this.p.length)):h=this.p:h=t;for(var f=h.length,c=this.c,l=c&&this.d,p=function(){var t=ft(h,a);if(67324752==t){s=1,u=a,v.d=null,v.c=0;var n=ht(h,a+6),r=ht(h,a+8),e=2048&n,o=8&n,l=ht(h,a+26),p=ht(h,a+28);if(f>a+30+l+p){var d=[];v.k.unshift(d),s=2;var g,y=ft(h,a+18),m=ft(h,a+22),b=rn(h.subarray(a+30,a+=30+l),!e),w=an(h,a,p,2,y,m,0),x=w[0],z=w[1];o&&(x=-1-w[3]),a+=p,v.c=x;var k={name:b,compression:r,start:function(){if(k.ondata||I(5),x){var t=i.o[r];t||k.ondata(I(14,"unknown compression type "+r,1),null,!1),(g=x<0?new t(b):new t(b,x,z)).ondata=function(t,n,r){k.ondata(t,n,r)};for(var n=0,e=d;n=0&&(k.size=x,k.originalSize=z),v.onfile(k)}return"break"}if(c){if(134695760==t)return u=a+=12+(-2==c&&8),s=3,v.c=0,"break";if(33639248==t)return u=a-=4,s=3,v.c=0,"break"}},v=this;a65558)return a(I(13,0,1),null),o;var h=ht(t,u+8);if(h){var f=h,c=ft(t,u+16),l=117853008==ft(t,u-20);if(l){var p=ft(t,u-12);(l=101075792==ft(t,p))&&(f=h=ft(t,p+32),c=ft(t,p+48))}for(var v=r&&r.filter,d=function(r){var i=sn(t,c,l),u=i[0],f=i[1],p=i[2],d=i[3],g=i[4],y=on(t,i[5]);c=g;var m=function(t,n){t?(o(),a(t,null)):(n&&(s[d]=n),--h||a(null,s))};if(!v||v({name:d,size:f,originalSize:p,compression:u}))if(u)if(8==u){var b=t.subarray(y,y+f);if(p<524288||f>.8*p)try{m(null,Tt(b,{out:new n(p)}))}catch(t){m(t,null)}else e.push(At(b,{size:p},m))}else m(I(14,"unknown compression type "+u,1),null);else m(null,D(t,y,y+f));else m(null,null)},g=0;g65558)&&I(13);var o=ht(t,e+8);if(!o)return{};var s=ft(t,e+16),a=117853008==ft(t,e-20);if(a){var u=ft(t,e-12);(a=101075792==ft(t,u))&&(o=ft(t,u+32),s=ft(t,u+48))}for(var h=r&&r.filter,f=0;f - Upload Platform + Upload Platform {{ version }} -
-

Upload Platform

-

Version: {{ version }}

+
+
+

FILE INTAKE / {{ version }}

+

Загрузка документов

+

Выберите отдельные файлы или целую папку. Архивы будут раскрыты автоматически.

+
+
+ + + + + + +
+

+
+ + + +
ПутьРазмерСтатус
+
+

+
+ + \ No newline at end of file diff --git a/upload/README.md b/upload/README.md new file mode 100644 index 0000000..e6b4999 --- /dev/null +++ b/upload/README.md @@ -0,0 +1,13 @@ +# Upload module + +Скопируйте каталог `upload/` в проект и подключите: + +- `create_upload_refs_blueprint(config)` к Flask-приложению; +- `initUploadTable(config)` из `frontend/table/init_upload_table.js`; +- `uploadViaVM(files, vmUploadUrl)` из `frontend/upload/upload_via_vm.js`. + +В конфигурации задаются URL ВМ-буфера, допустимые расширения, лимиты и параметры +повторных попыток pull. Backend-модуль хранит файлы сессии в памяти; обработка +файлов остаётся ответственностью приложения, которое интегрирует этот модуль. + +Для ZIP перед ES-модулями загрузите локальный `fflate` из `site/static/vendor/`. \ No newline at end of file diff --git a/upload/__init__.py b/upload/__init__.py new file mode 100644 index 0000000..f510179 --- /dev/null +++ b/upload/__init__.py @@ -0,0 +1 @@ +"""Переиспользуемые слои загрузки файлов.""" \ No newline at end of file diff --git a/upload/backend/__init__.py b/upload/backend/__init__.py new file mode 100644 index 0000000..29d32d6 --- /dev/null +++ b/upload/backend/__init__.py @@ -0,0 +1 @@ +"""Backend-модули переиспользуемого слоя загрузки.""" \ No newline at end of file diff --git a/upload/backend/session/__init__.py b/upload/backend/session/__init__.py new file mode 100644 index 0000000..ac0f436 --- /dev/null +++ b/upload/backend/session/__init__.py @@ -0,0 +1,19 @@ +"""Самодостаточное in-memory хранилище сессий.""" + +from .add_file import add_file +from .cleanup import cleanup +from .create_session import create_session +from .get_files import file_count, get_files +from .state import MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS, configure + +__all__ = [ + "create_session", + "add_file", + "get_files", + "file_count", + "cleanup", + "configure", + "TTL_SECONDS", + "MAX_FILE_BYTES", + "MAX_SESSION_BYTES", +] \ No newline at end of file diff --git a/upload/backend/session/add_file.py b/upload/backend/session/add_file.py new file mode 100644 index 0000000..ba77f89 --- /dev/null +++ b/upload/backend/session/add_file.py @@ -0,0 +1,17 @@ +"""Добавление файла в сессию.""" + +from . import state + + +def add_file(sid: str, filename: str, content: bytes) -> bool: + """Добавить файл, если сессия существует и общий лимит не превышен.""" + + with state._lock: + session = state._sessions.get(sid) + if not session: + return False + total = sum(len(item_content) for _, item_content in session["files"]) + if total + len(content) > state.MAX_SESSION_BYTES: + return False + session["files"].append((filename, content)) + return True \ No newline at end of file diff --git a/upload/backend/session/cleanup.py b/upload/backend/session/cleanup.py new file mode 100644 index 0000000..2b1774f --- /dev/null +++ b/upload/backend/session/cleanup.py @@ -0,0 +1,12 @@ +"""Удаление сессии.""" + +from .state import _lock, _sessions + + +def cleanup(sid: str): + """Удалить сессию и остановить её TTL-таймер.""" + + with _lock: + session = _sessions.pop(sid, None) + if session and session.get("timer"): + session["timer"].cancel() \ No newline at end of file diff --git a/upload/backend/session/create_session.py b/upload/backend/session/create_session.py new file mode 100644 index 0000000..df2acca --- /dev/null +++ b/upload/backend/session/create_session.py @@ -0,0 +1,19 @@ +"""Создание сессии.""" + +import threading +import uuid + +from .state import _lock, _sessions, _start_timer + + +def create_session() -> str: + """Создать сессию и вернуть её уникальный идентификатор.""" + + sid = uuid.uuid4().hex + with _lock: + _sessions[sid] = { + "files": [], + "timer": _start_timer(sid), + "cancel": threading.Event(), + } + return sid \ No newline at end of file diff --git a/upload/backend/session/get_files.py b/upload/backend/session/get_files.py new file mode 100644 index 0000000..28c17bc --- /dev/null +++ b/upload/backend/session/get_files.py @@ -0,0 +1,21 @@ +"""Чтение файлов сессии.""" + +from typing import List, Optional, Tuple + +from .state import _lock, _sessions + + +def get_files(sid: str) -> Optional[List[Tuple[str, bytes]]]: + """Вернуть файлы сессии или None, если сессия не найдена.""" + + with _lock: + session = _sessions.get(sid) + return list(session["files"]) if session else None + + +def file_count(sid: str) -> int: + """Вернуть количество файлов в сессии.""" + + with _lock: + session = _sessions.get(sid) + return len(session["files"]) if session else 0 \ No newline at end of file diff --git a/upload/backend/session/state.py b/upload/backend/session/state.py new file mode 100644 index 0000000..cabd9ce --- /dev/null +++ b/upload/backend/session/state.py @@ -0,0 +1,37 @@ +"""Общее состояние сессий, блокировка, лимиты и TTL.""" + +import threading + + +TTL_SECONDS = 30 * 60 +MAX_FILE_BYTES = 50 * 1024 * 1024 +MAX_SESSION_BYTES = 500 * 1024 * 1024 + +_sessions: dict = {} +_lock = threading.Lock() + + +def _start_timer(sid: str) -> threading.Timer: + """Запустить таймер автоочистки сессии через TTL.""" + + def _clean(): + with _lock: + _sessions.pop(sid, None) + + timer = threading.Timer(TTL_SECONDS, _clean) + timer.daemon = True + timer.start() + return timer + + +def configure(max_file_bytes: int = None, max_session_bytes: int = None, + ttl_seconds: int = None): + """Переопределить лимиты и TTL из конфигурации приложения.""" + + global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS + if max_file_bytes is not None: + MAX_FILE_BYTES = max_file_bytes + if max_session_bytes is not None: + MAX_SESSION_BYTES = max_session_bytes + if ttl_seconds is not None: + TTL_SECONDS = ttl_seconds \ No newline at end of file diff --git a/upload/backend/upload_refs/__init__.py b/upload/backend/upload_refs/__init__.py new file mode 100644 index 0000000..c27cad5 --- /dev/null +++ b/upload/backend/upload_refs/__init__.py @@ -0,0 +1,15 @@ +"""Переиспользуемый backend-слой закачки через ВМ.""" + +from .blueprint import create_upload_refs_blueprint +from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX +from .pull_file import pull_file +from .safe_name import safe_name + +__all__ = [ + "create_upload_refs_blueprint", + "safe_name", + "pull_file", + "PULL_RETRIES", + "PULL_RETRY_DELAY", + "VM_UPLOAD_PREFIX", +] \ No newline at end of file diff --git a/upload/backend/upload_refs/blueprint.py b/upload/backend/upload_refs/blueprint.py new file mode 100644 index 0000000..cf4fed1 --- /dev/null +++ b/upload/backend/upload_refs/blueprint.py @@ -0,0 +1,79 @@ +"""Blueprint слоя закачки ссылок через ВМ-буфер.""" + +import logging + +import requests +from flask import Blueprint, jsonify, request + +from ..session import (MAX_FILE_BYTES, add_file, configure, create_session, + file_count, get_files) +from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX +from .pull_file import pull_file +from .safe_name import safe_name + + +log = logging.getLogger("upload.upload_refs") + + +def create_upload_refs_blueprint(cfg: dict) -> Blueprint: + """Создать Blueprint с POST ``/upload_refs``.""" + + cfg = cfg or {} + prefix = cfg.get("apiPrefix", "/api") + vm_prefix = cfg.get("vmUploadUrl", VM_UPLOAD_PREFIX) + max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES) + pull_retries = cfg.get("pullRetries", PULL_RETRIES) + pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY) + pull_timeout = cfg.get("pullTimeout", 120) + configure(max_session_bytes=cfg.get("maxSessionBytes")) + + blueprint = Blueprint("upload_refs", __name__, url_prefix=prefix) + + @blueprint.route("/upload_refs", methods=["POST"]) + def upload_refs(): + data = request.get_json(silent=True) or {} + sid = data.get("session") or create_session() + refs = data.get("files") or [] + if not isinstance(refs, list) or not refs: + return jsonify({"ok": False, "error": "No files"}), 400 + + added = 0 + try: + with requests.Session() as client: + for ref in refs: + if not isinstance(ref, dict): + continue + name = safe_name(ref.get("name", "")) + url = ref.get("url") + if not name or not isinstance(url, str) or not url.startswith(vm_prefix): + continue + if (ref.get("size") or 0) > max_file_bytes: + _delete(client, url) + continue + content = pull_file( + client, url, max_file_bytes, pull_retries, pull_delay, + pull_timeout, sid=sid, name=name, + ) + if not add_file(sid, name, content): + if get_files(sid) is None: + return jsonify({"ok": False, "error": "Session not found"}), 404 + _delete(client, url) + continue + _delete(client, url) + added += 1 + except Exception as error: + log.error("upload_refs: pull error sid=%s: %r", sid, error) + return jsonify({"ok": False, "error": "Pull failed: %s" % error}), 502 + + return jsonify({"ok": True, "session": sid, "count": file_count(sid)}) + + return blueprint + + +def _delete(client, url: str): + """Удалить временный объект на ВМ, не ломая основной запрос при сбое.""" + + try: + client.delete(url, timeout=10) + except Exception: + log.warning("upload_refs: could not delete VM object url=%r", url) \ No newline at end of file diff --git a/upload/backend/upload_refs/config.py b/upload/backend/upload_refs/config.py new file mode 100644 index 0000000..fe1b637 --- /dev/null +++ b/upload/backend/upload_refs/config.py @@ -0,0 +1,5 @@ +"""Значения по умолчанию для pull из ВМ-буфера.""" + +PULL_RETRIES = 3 +PULL_RETRY_DELAY = 2 +VM_UPLOAD_PREFIX = "https://example.invalid/upload/" \ No newline at end of file diff --git a/upload/backend/upload_refs/pull_file.py b/upload/backend/upload_refs/pull_file.py new file mode 100644 index 0000000..6ca49b1 --- /dev/null +++ b/upload/backend/upload_refs/pull_file.py @@ -0,0 +1,38 @@ +"""Исходящий pull файла из ВМ-буфера с ретраями и лимитом размера.""" + +import logging +import time + +from .config import PULL_RETRIES, PULL_RETRY_DELAY + + +log = logging.getLogger("upload.upload_refs.pull") + + +def pull_file(client, url: str, max_bytes: int, + retries: int = PULL_RETRIES, delay: float = PULL_RETRY_DELAY, + timeout: int = 120, sid: str = None, name: str = None) -> bytes: + """Забрать файл потоково, не принимая тело больше ``max_bytes``.""" + + last_error = None + for attempt in range(retries): + try: + response = client.get(url, stream=True, timeout=timeout) + response.raise_for_status() + chunks = [] + total = 0 + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + raise ValueError("file exceeds maxFileBytes") + chunks.append(chunk) + return b"".join(chunks) + except Exception as error: + last_error = error + log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r", + attempt + 1, retries, sid, name, error) + if attempt + 1 < retries: + time.sleep(delay) + raise last_error if last_error else RuntimeError("pull failed") \ No newline at end of file diff --git a/upload/backend/upload_refs/safe_name.py b/upload/backend/upload_refs/safe_name.py new file mode 100644 index 0000000..be7f0e7 --- /dev/null +++ b/upload/backend/upload_refs/safe_name.py @@ -0,0 +1,13 @@ +"""Санитизация имени файла с сохранением подпапок.""" + + +def safe_name(name: str) -> str: + """Вернуть безопасное относительное имя или пустую строку.""" + + if not isinstance(name, str) or not name: + return "" + parts = [part for part in name.replace("\\", "/").split("/") + if part and part != "."] + if not parts or any(part == ".." for part in parts): + return "" + return "/".join(parts) \ No newline at end of file diff --git a/upload/config.example.json b/upload/config.example.json new file mode 100644 index 0000000..5e14973 --- /dev/null +++ b/upload/config.example.json @@ -0,0 +1,9 @@ +{ + "vmUploadUrl": "https://example.invalid/upload/", + "allowedExt": [".pdf", ".doc", ".docx", ".txt", ".md"], + "maxFileBytes": 52428800, + "maxSessionBytes": 524288000, + "apiPrefix": "/api", + "pullRetries": 3, + "pullRetryDelay": 2 +} \ No newline at end of file diff --git a/upload/frontend/table/add_file_with_dedup.js b/upload/frontend/table/add_file_with_dedup.js new file mode 100644 index 0000000..75c6331 --- /dev/null +++ b/upload/frontend/table/add_file_with_dedup.js @@ -0,0 +1,27 @@ +export function addFileWithDedup(state, file, cfg) { + const originalName = file.name; + const existing = state.fileMeta.get(originalName); + let name = originalName; + + if (existing) { + if (existing.size === file.size) return false; + const dot = originalName.lastIndexOf('.'); + const base = dot > 0 ? originalName.slice(0, dot) : originalName; + const extension = dot > 0 ? originalName.slice(dot) : ''; + let suffix = 2; + while (state.fileMeta.has(`${base}_${suffix}${extension}`)) suffix += 1; + name = `${base}_${suffix}${extension}`; + } + + const storedFile = new File([file], name, { lastModified: file.lastModified }); + state.fileMeta.set(name, { size: storedFile.size }); + const includedBytes = state.files.reduce( + (total, item) => total + (state.overNames.has(item.name) ? 0 : item.size), 0, + ); + if (storedFile.size > cfg.maxFileBytes + || includedBytes + storedFile.size > cfg.maxSessionBytes) { + state.overNames.add(name); + } + state.files.push(storedFile); + return true; +} \ No newline at end of file diff --git a/upload/frontend/table/esc.js b/upload/frontend/table/esc.js new file mode 100644 index 0000000..664a404 --- /dev/null +++ b/upload/frontend/table/esc.js @@ -0,0 +1,9 @@ +export function esc(value) { + return String(value).replace(/[&<>"']/g, (character) => ({ + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', + }[character])); +} \ No newline at end of file diff --git a/upload/frontend/table/fs.js b/upload/frontend/table/fs.js new file mode 100644 index 0000000..705701d --- /dev/null +++ b/upload/frontend/table/fs.js @@ -0,0 +1,5 @@ +export function fs(bytes) { + return bytes < 1024 ? `${bytes} B` + : bytes < 1048576 ? `${(bytes / 1024).toFixed(1)} KB` + : `${(bytes / 1048576).toFixed(1)} MB`; +} \ No newline at end of file diff --git a/upload/frontend/table/init_upload_table.js b/upload/frontend/table/init_upload_table.js new file mode 100644 index 0000000..8955a02 --- /dev/null +++ b/upload/frontend/table/init_upload_table.js @@ -0,0 +1,44 @@ +import { addFiles } from './on_files_change.js'; +import { onFilesChange } from './on_files_change.js'; +import { onFolderChange } from './on_folder_change.js'; +import { render } from './render.js'; +import { setStatus } from './set_status.js'; + +export function initUploadTable(cfg) { + const elements = { + fileInputEl: cfg.fileInputEl, + folderInputEl: cfg.folderInputEl, + tableBodyEl: cfg.tableBodyEl, + countEl: cfg.countEl, + uploadBtnEl: cfg.uploadBtnEl, + }; + const state = { files: [], fileMeta: new Map(), overNames: new Set(), busy: false }; + elements.fileInputEl.addEventListener('change', onFilesChange(state, cfg, elements)); + elements.folderInputEl.addEventListener('change', onFolderChange(state, cfg, elements)); + + const api = { + pickFiles: () => elements.fileInputEl.click(), + pickFolder: () => elements.folderInputEl.click(), + addFiles: (files) => addFiles(state, cfg, files, elements), + getFiles: () => state.files + .filter((file) => !state.overNames.has(file.name)) + .map((file) => ({ path: file.name, name: file.name, size: file.size, file })), + getOverNames: () => new Set(state.overNames), + setStatus: (path, html) => setStatus(path, html, state, elements), + render: () => render(state, elements), + clear: () => { + state.files = []; + state.fileMeta.clear(); + state.overNames.clear(); + render(state, elements); + }, + setBusy: (busy) => { + state.busy = busy; + elements.fileInputEl.disabled = busy; + elements.folderInputEl.disabled = busy; + elements.uploadBtnEl.disabled = busy || api.getFiles().length === 0; + }, + }; + api.render(); + return api; +} \ No newline at end of file diff --git a/upload/frontend/table/on_files_change.js b/upload/frontend/table/on_files_change.js new file mode 100644 index 0000000..5918bd8 --- /dev/null +++ b/upload/frontend/table/on_files_change.js @@ -0,0 +1,29 @@ +import { listZipFiles } from '../zip/list_zip_files.js'; +import { addFileWithDedup } from './add_file_with_dedup.js'; +import { render } from './render.js'; + +export async function addFiles(state, cfg, files, elements) { + for (const file of Array.from(files)) { + if (!file.name.toLowerCase().endsWith('.zip')) { + addFileWithDedup(state, file, cfg); + continue; + } + try { + const extracted = await listZipFiles(file, cfg.allowedExt); + if (extracted.length) { + extracted.forEach((item) => addFileWithDedup(state, item, cfg)); + } else { + addFileWithDedup(state, file, cfg); + } + } catch (error) { + addFileWithDedup(state, file, cfg); + } + } + render(state, elements); +} + +export function onFilesChange(state, cfg, elements) { + return () => { + if (!state.busy) addFiles(state, cfg, elements.fileInputEl.files, elements); + }; +} \ No newline at end of file diff --git a/upload/frontend/table/on_folder_change.js b/upload/frontend/table/on_folder_change.js new file mode 100644 index 0000000..26f446f --- /dev/null +++ b/upload/frontend/table/on_folder_change.js @@ -0,0 +1,37 @@ +import { listZipFiles } from '../zip/list_zip_files.js'; +import { addFileWithDedup } from './add_file_with_dedup.js'; +import { render } from './render.js'; + +export function onFolderChange(state, cfg, elements) { + return async () => { + if (state.busy) return; + for (const file of Array.from(elements.folderInputEl.files)) { + const parts = (file.webkitRelativePath || file.name).split('/'); + const relativePath = parts.slice(1).join('/') || file.name; + const lowerPath = relativePath.toLowerCase(); + const directory = relativePath.includes('/') + ? relativePath.slice(0, relativePath.lastIndexOf('/')) : ''; + if (lowerPath.endsWith('.zip')) { + try { + const extracted = await listZipFiles(file, cfg.allowedExt); + if (extracted.length) { + extracted.forEach((item) => addFileWithDedup(state, new File([item], + directory ? `${directory}/${item.name}` : item.name, + { lastModified: item.lastModified }), cfg)); + } else { + addFileWithDedup(state, new File([file], relativePath, + { lastModified: file.lastModified }), cfg); + } + } catch (error) { + addFileWithDedup(state, new File([file], relativePath, + { lastModified: file.lastModified }), cfg); + } + } else if (cfg.allowedExt.some((extension) => lowerPath.endsWith(extension))) { + addFileWithDedup(state, new File([file], relativePath, + { lastModified: file.lastModified }), cfg); + } + } + elements.folderInputEl.value = ''; + render(state, elements); + }; +} \ No newline at end of file diff --git a/upload/frontend/table/render.js b/upload/frontend/table/render.js new file mode 100644 index 0000000..98cc42e --- /dev/null +++ b/upload/frontend/table/render.js @@ -0,0 +1,22 @@ +import { esc } from './esc.js'; +import { fs } from './fs.js'; + +export function render(state, elements) { + if (!state.files.length) { + elements.tableBodyEl.innerHTML = 'Нет выбранных файлов'; + } else { + elements.tableBodyEl.innerHTML = state.files.map((file, index) => { + const over = state.overNames.has(file.name); + const status = over ? 'не учитывается (лимит)' : 'готов'; + return `${esc(file.name)}` + + `${fs(file.size)}${status}`; + }).join(''); + } + const included = state.files.filter((file) => !state.overNames.has(file.name)); + const overCount = state.files.length - included.length; + const size = included.reduce((total, file) => total + file.size, 0); + elements.countEl.textContent = `${included.length} учитываются` + + (overCount ? ` + ${overCount} свыше лимита` : '') + + ` · ${fs(size)}`; + elements.uploadBtnEl.disabled = included.length === 0; +} \ No newline at end of file diff --git a/upload/frontend/table/set_status.js b/upload/frontend/table/set_status.js new file mode 100644 index 0000000..5f91ab3 --- /dev/null +++ b/upload/frontend/table/set_status.js @@ -0,0 +1,5 @@ +export function setStatus(path, html, state, elements) { + const index = state.files.findIndex((file) => file.name === path); + const cell = elements.tableBodyEl.querySelector(`#st-${index}`); + if (cell) cell.innerHTML = html; +} \ No newline at end of file diff --git a/upload/frontend/upload/put_to_vm.js b/upload/frontend/upload/put_to_vm.js new file mode 100644 index 0000000..915b153 --- /dev/null +++ b/upload/frontend/upload/put_to_vm.js @@ -0,0 +1,19 @@ +export function putToVm(file, url, options = {}) { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open('PUT', url); + xhr.timeout = options.timeoutMs || 300000; + xhr.upload.onprogress = (event) => { + if (event.lengthComputable && options.onProgress) { + options.onProgress(Math.round(event.loaded / event.total * 100)); + } + }; + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) resolve(); + else reject(new Error(`ВМ: HTTP ${xhr.status}`)); + }; + xhr.onerror = () => reject(new Error('Сеть (ВМ)')); + xhr.ontimeout = () => reject(new Error('Таймаут загрузки на ВМ')); + xhr.send(file); + }); +} \ No newline at end of file diff --git a/upload/frontend/upload/upload_via_vm.js b/upload/frontend/upload/upload_via_vm.js new file mode 100644 index 0000000..f923d33 --- /dev/null +++ b/upload/frontend/upload/upload_via_vm.js @@ -0,0 +1,33 @@ +import { putToVm } from './put_to_vm.js'; + +export async function uploadViaVM(files, vmUploadUrl, options = {}) { + const token = crypto.randomUUID(); + const refs = []; + for (let index = 0; index < files.length; index += 1) { + const file = files[index]; + const url = `${vmUploadUrl}${token}_${index}`; + options.onUploadStatus?.(`Загрузка на ВМ ${index + 1}/${files.length}: ${file.name}`); + try { + await putToVm(file, url, { + onProgress: (percent) => options.onStatus?.(file.name, `${percent}%`), + }); + refs.push({ name: file.name, size: file.size, url }); + } catch (error) { + options.onStatus?.(file.name, `Ошибка: ${error.message}`); + return { ok: false, error: `Ошибка загрузки на ВМ: ${error.message}` }; + } + } + + try { + const response = await fetch(`${options.apiBase || ''}/api/upload_refs`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session: options.session || '', files: refs }), + }); + const data = await response.json(); + if (!response.ok || !data.ok) throw new Error(data.error || `HTTP ${response.status}`); + return { ok: true, session: data.session, count: data.count || refs.length }; + } catch (error) { + return { ok: false, error: `Ошибка передачи ссылок: ${error.message}` }; + } +} \ No newline at end of file diff --git a/upload/frontend/zip/list_zip_files.js b/upload/frontend/zip/list_zip_files.js new file mode 100644 index 0000000..4da6d3f --- /dev/null +++ b/upload/frontend/zip/list_zip_files.js @@ -0,0 +1,32 @@ +// Рекурсивно получить из ZIP только файлы с разрешёнными расширениями. + +function extensionAllowed(name, allowedExt) { + const lowerName = name.toLowerCase(); + return allowedExt.some((extension) => lowerName.endsWith(extension.toLowerCase())); +} + +function makeFile(data, name) { + return new File([data], name); +} + +async function listEntries(data, prefix, allowedExt, depth) { + if (depth > 20) throw new Error('Слишком глубокая вложенность ZIP'); + const entries = fflate.unzipSync(data); + const files = []; + + for (const [entryName, entryData] of Object.entries(entries)) { + if (entryName.endsWith('/')) continue; + const path = prefix ? `${prefix}/${entryName}` : entryName; + if (entryName.toLowerCase().endsWith('.zip')) { + files.push(...await listEntries(entryData, path, allowedExt, depth + 1)); + } else if (extensionAllowed(entryName, allowedExt)) { + files.push(makeFile(entryData, path)); + } + } + return files; +} + +export async function listZipFiles(file, allowedExt) { + const data = new Uint8Array(await file.arrayBuffer()); + return listEntries(data, '', allowedExt, 0); +} \ No newline at end of file