process.cfm: один запрос вместо N+1, skip already extracted, DELETE before INSERT, prompt fix (v1.0.49)
This commit is contained in:
+1
-1
@@ -78,7 +78,7 @@
|
||||
4. Верни ТОЛЬКО JSON, без пояснений.
|
||||
|
||||
Пример ответа:
|
||||
{"rows":[{"row_num":1,"name":"Аренда стойко-места","price":213905.44,"qty":3,"sum":641716.32,"date_start":"2026-04-01"}],"unresolved":[]}
|
||||
{{"rows":[{{"row_num":1,"name":"Аренда стойко-места","price":213905.44,"qty":3,"sum":641716.32,"date_start":"2026-04-01"}}],"unresolved":[]}}
|
||||
|
||||
Текст документа:
|
||||
---
|
||||
|
||||
@@ -61,7 +61,7 @@
|
||||
<body>
|
||||
<div class="topbar">
|
||||
<img src="/nubes-logo.svg" alt="Nubes">
|
||||
<span class="title">Сверка договоров — LLM <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.47 — Lucee</span></span>
|
||||
<span class="title">Сверка договоров — LLM <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.49 — Lucee</span></span>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
@@ -340,30 +340,31 @@ document.getElementById('llmBtn').addEventListener('click', async function() {
|
||||
diffStatus.textContent = '⏳ ' + ((Date.now() - llmStart) / 1000).toFixed(1) + 'с';
|
||||
}, 200);
|
||||
|
||||
// 1. Extract for each supplement
|
||||
// ОДИН запрос — всё внутри: экстракция + сравнение
|
||||
try {
|
||||
var resp = await fetch('/api.cfm?action=query&sql=' + encodeURIComponent("SELECT id FROM supplements WHERE contract_id='" + contractId + "' ORDER BY created_at"));
|
||||
var resp = await fetch('/process.cfm?contract_id=' + contractId);
|
||||
var data = await resp.json();
|
||||
if (data.OK && data.ROWS) {
|
||||
for (var i = 0; i < data.ROWS.length; i++) {
|
||||
try {
|
||||
var er = await fetch('/extractor.cfm?supplement_id=' + data.ROWS[i].id);
|
||||
var ed = await er.json();
|
||||
if (ed.OK) diffBody.innerHTML += '<div style="font-size:12px;color:var(--green);">✓ ' + ed.ROWS_SAVED + ' строк</div>';
|
||||
else diffBody.innerHTML += '<div style="font-size:12px;color:var(--destructive);">✗ ' + (ed.ERROR||'') + '</div>';
|
||||
} catch(e) {}
|
||||
}
|
||||
}
|
||||
} catch(e) {}
|
||||
|
||||
// 2. Differ
|
||||
diffBody.innerHTML += '<div style="margin-top:8px;font-weight:600;">📊 Сравнение:</div>';
|
||||
try {
|
||||
var resp = await fetch('/differ.cfm?contract_id=' + contractId);
|
||||
var data = await resp.json();
|
||||
if (data.OK && data.ALL_CHANGES) {
|
||||
var all = data.ALL_CHANGES;
|
||||
if (data.OK) {
|
||||
// Показать результаты экстракции
|
||||
var extr = data.EXTRACTED || [];
|
||||
diffBody.innerHTML = '';
|
||||
extr.forEach(function(e) {
|
||||
if (e.ERROR) {
|
||||
diffBody.innerHTML += '<div style="font-size:12px;color:var(--destructive);">✗ ' + (e.FILENAME||e.SUPPLEMENT_ID) + ': ' + e.ERROR + '</div>';
|
||||
} else if (e.SKIPPED) {
|
||||
diffBody.innerHTML += '<div style="font-size:12px;color:var(--muted);">⏭ ' + (e.FILENAME||e.SUPPLEMENT_ID) + ' (уже извлечено, ' + e.ROWS_SAVED + ' строк)</div>';
|
||||
} else {
|
||||
diffBody.innerHTML += '<div style="font-size:12px;color:var(--green);">✓ ' + (e.FILENAME||e.SUPPLEMENT_ID) + ': ' + e.ROWS_SAVED + ' строк</div>';
|
||||
}
|
||||
});
|
||||
|
||||
// Сравнение
|
||||
diffBody.innerHTML += '<div style="margin-top:8px;font-weight:600;">📊 Сравнение:</div>';
|
||||
var all = data.ALL_CHANGES || [];
|
||||
if (all.length === 0) {
|
||||
diffBody.innerHTML += '<div style="color:var(--muted);font-size:12px;">Нет допников для сравнения.</div>';
|
||||
} else {
|
||||
all.forEach(function(ch) {
|
||||
var s = ch.SUMMARY || {};
|
||||
diffBody.innerHTML += '<div style="font-weight:600;margin-top:12px;">📄 Допник (' + ch.TYPE + ')</div>';
|
||||
@@ -384,10 +385,13 @@ document.getElementById('llmBtn').addEventListener('click', async function() {
|
||||
diffBody.innerHTML += tbl;
|
||||
}
|
||||
});
|
||||
} else {
|
||||
diffBody.innerHTML += '<div style="color:var(--destructive);">✗ ' + (data.ERROR||'') + '</div>';
|
||||
}
|
||||
} catch(e) { diffBody.innerHTML += '<div style="color:var(--destructive);">✗ ' + e.message + '</div>'; }
|
||||
} else {
|
||||
diffBody.innerHTML = '<div style="color:var(--destructive);">✗ ' + (data.ERROR||'Неизвестная ошибка') + '</div>';
|
||||
}
|
||||
} catch(e) {
|
||||
diffBody.innerHTML = '<div style="color:var(--destructive);">✗ ' + e.message + '</div>';
|
||||
}
|
||||
|
||||
clearInterval(timerInterval);
|
||||
diffStatus.textContent = '✓ ' + ((Date.now() - llmStart) / 1000).toFixed(1) + 'с';
|
||||
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
<cfsetting showdebugoutput="no" enablecfoutputonly="true" requesttimeout="600">
|
||||
<cfheader name="Content-Type" value="application/json; charset=utf-8">
|
||||
|
||||
<cfparam name="url.contract_id" default="">
|
||||
<cfset result = {OK: false, ERROR: ""}>
|
||||
|
||||
<cftry>
|
||||
<cfif NOT len(url.contract_id)>
|
||||
<cfset result.ERROR = "contract_id required">
|
||||
<cfelse>
|
||||
<!--- Все допники договора с parsed_text --->
|
||||
<cfquery name="supps" datasource="baza">
|
||||
SELECT s.id as supp_id, s.type, d.id as doc_id, d.filename, d.elements_json, d.parsed_text
|
||||
FROM supplements s
|
||||
JOIN documents d ON s.document_id = d.id
|
||||
WHERE s.contract_id = <cfqueryparam value="#url.contract_id#" cfsqltype="cf_sql_varchar">
|
||||
AND d.parsed_text IS NOT NULL
|
||||
ORDER BY s.created_at
|
||||
</cfquery>
|
||||
|
||||
<cfif supps.recordCount EQ 0>
|
||||
<cfset result.ERROR = "Нет распарсенных файлов. Сначала нажмите Парсинг.">
|
||||
<cfelse>
|
||||
<cfset extractedInfo = []>
|
||||
<cfset extractedRows = {}> <!--- supp_id → [rows] --->
|
||||
|
||||
<!--- Промпт из БД --->
|
||||
<cfset promptText = "">
|
||||
<cfquery name="pq" datasource="baza">
|
||||
SELECT prompt_text FROM prompts WHERE contract_id = <cfqueryparam value="#url.contract_id#" cfsqltype="cf_sql_varchar">
|
||||
</cfquery>
|
||||
<cfif pq.recordCount GT 0 AND len(pq.prompt_text)>
|
||||
<cfset promptText = pq.prompt_text>
|
||||
</cfif>
|
||||
|
||||
<!--- DEFAULT_PROMPT --->
|
||||
<cfset defaultPrompt = 'Ты — анализатор договоров. Ниже текст спецификации услуг из договора облачного провайдера.
|
||||
|
||||
Найди ВСЕ таблицы со списком услуг. Извлеки каждую строку в JSON-массив.
|
||||
|
||||
Для каждой строки верни:
|
||||
- row_num: номер по порядку (целое число)
|
||||
- name: полное наименование услуги (весь текст из ячейки)
|
||||
- price: цена за единицу в рублях (число, из колонки "Цена")
|
||||
- qty: количество/объём (число, из колонки "Объем")
|
||||
- sum: итоговая сумма (число, из колонки "Сумма")
|
||||
- date_start: дата начала оказания (строка YYYY-MM-DD)
|
||||
|
||||
ПРАВИЛА:
|
||||
1. Пропускай итоговые строки ("Итого...") и строки с подписями.
|
||||
2. Если ячейка пустая — ставь null.
|
||||
3. Если не можешь определить значение — ставь null и добавь в unresolved.
|
||||
4. Верни ТОЛЬКО JSON, без пояснений.
|
||||
|
||||
Пример ответа:
|
||||
{{"rows":[{{"row_num":1,"name":"Аренда стойко-места","price":213905.44,"qty":3,"sum":641716.32,"date_start":"2026-04-01"}}],"unresolved":[]}}
|
||||
|
||||
Текст документа:
|
||||
---
|
||||
{parsed_text}
|
||||
---'>
|
||||
|
||||
<!--- LLM API --->
|
||||
<cfset apiKey = "sk-ucI5YvOticoOQ9Kuj5K9mQ">
|
||||
|
||||
<!--- Шаг 1: LLM-извлечение для каждого допника --->
|
||||
<cfloop query="supps">
|
||||
<cfset sid = supps.supp_id>
|
||||
|
||||
<!--- Проверить: уже извлечено? --->
|
||||
<cfquery name="ex" datasource="baza">
|
||||
SELECT row_num, name, price, qty, sum, date_start
|
||||
FROM spec_rows WHERE supplement_id = <cfqueryparam value="#sid#" cfsqltype="cf_sql_varchar">
|
||||
ORDER BY row_num
|
||||
</cfquery>
|
||||
|
||||
<cfif ex.recordCount GT 0>
|
||||
<!--- Уже извлечено — загрузить существующие --->
|
||||
<cfset rows = []>
|
||||
<cfloop query="ex">
|
||||
<cfset arrayAppend(rows, {ROW_NUM: ex.row_num, NAME: ex.name, PRICE: ex.price, QTY: ex.qty, SUM: ex.sum, DATE_START: ex.date_start})>
|
||||
</cfloop>
|
||||
<cfset extractedRows[sid] = rows>
|
||||
<cfset arrayAppend(extractedInfo, {SUPPLEMENT_ID: sid, FILENAME: supps.filename, ROWS_SAVED: ex.recordCount, SKIPPED: true})>
|
||||
<cfcontinue>
|
||||
</cfif>
|
||||
|
||||
<!--- textify: elements → текст --->
|
||||
<cfset elements = deserializeJSON(supps.elements_json)>
|
||||
<cfset textLines = []>
|
||||
<cfloop array="#elements#" index="el">
|
||||
<cfif el.type EQ "paragraph">
|
||||
<cfset prefix = len(el.style) ? "[#el.style#] " : "">
|
||||
<cfset arrayAppend(textLines, prefix & el.text)>
|
||||
<cfelseif el.type EQ "table">
|
||||
<cfset rows = el.rows>
|
||||
<cfif arrayLen(rows) GT 0>
|
||||
<cfset ncols = arrayLen(rows[1])>
|
||||
<cfset arrayAppend(textLines, "--- Таблица (" & arrayLen(rows) & "×" & ncols & ") ---")>
|
||||
<cfloop array="#rows#" index="row">
|
||||
<cfset cells = []>
|
||||
<cfloop from="1" to="#ncols#" index="ci">
|
||||
<cfset val = "">
|
||||
<cfif ci LTE arrayLen(row)>
|
||||
<cfset val = replace(replace(toString(row[ci]), chr(10), " ", "ALL"), "|", "\|", "ALL")>
|
||||
</cfif>
|
||||
<cfset arrayAppend(cells, val)>
|
||||
</cfloop>
|
||||
<cfset arrayAppend(textLines, "| " & arrayToList(cells, " | ") & " |")>
|
||||
</cfloop>
|
||||
<cfset arrayAppend(textLines, "")>
|
||||
</cfif>
|
||||
</cfif>
|
||||
</cfloop>
|
||||
<cfset parsedText = arrayToList(textLines, chr(10))>
|
||||
|
||||
<!--- Промпт --->
|
||||
<cfset prompt = replace(len(promptText) ? promptText : defaultPrompt, "{parsed_text}", parsedText, "ALL")>
|
||||
|
||||
<!--- LLM --->
|
||||
<cfset llmPayload = {
|
||||
"model": "gpt-oss-120b",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 8000,
|
||||
"temperature": 0.1
|
||||
}>
|
||||
|
||||
<cfhttp url="https://api.aillm.ru/v1/chat/completions" method="POST" timeout="120">
|
||||
<cfhttpparam type="header" name="Authorization" value="Bearer #apiKey#">
|
||||
<cfhttpparam type="header" name="Content-Type" value="application/json">
|
||||
<cfhttpparam type="body" value="#serializeJSON(llmPayload)#">
|
||||
</cfhttp>
|
||||
|
||||
<cfif NOT (cfhttp.statusCode EQ 200 OR left(cfhttp.statusCode, 3) EQ "200")>
|
||||
<cfset arrayAppend(extractedInfo, {SUPPLEMENT_ID: sid, FILENAME: supps.filename, ERROR: "LLM HTTP #cfhttp.statusCode#"})>
|
||||
<cfcontinue>
|
||||
</cfif>
|
||||
|
||||
<cfset llmResp = deserializeJSON(cfhttp.fileContent)>
|
||||
<cfset llmText = llmResp.choices[1].message.content>
|
||||
|
||||
<!--- Вытащить JSON из ответа --->
|
||||
<cfset jsonText = llmText>
|
||||
<cfset p1 = find("```json", jsonText)>
|
||||
<cfif p1 GT 0>
|
||||
<cfset jsonText = mid(jsonText, p1 + 7, len(jsonText))>
|
||||
<cfset p2 = find("```", jsonText)>
|
||||
<cfif p2 GT 0>
|
||||
<cfset jsonText = left(jsonText, p2 - 1)>
|
||||
</cfif>
|
||||
<cfelse>
|
||||
<cfset p1 = find("{", jsonText)>
|
||||
<cfset p2 = find("}", reverse(jsonText))>
|
||||
<cfif p1 GT 0 AND p2 GT 0>
|
||||
<cfset jsonText = mid(jsonText, p1, len(jsonText) - p2 - p1 + 2)>
|
||||
</cfif>
|
||||
</cfif>
|
||||
<cfset jsonText = trim(jsonText)>
|
||||
|
||||
<cftry>
|
||||
<cfset extracted = deserializeJSON(jsonText)>
|
||||
<cfset rows = []>
|
||||
<cfif structKeyExists(extracted, "rows") AND isArray(extracted.rows)>
|
||||
<cfset rows = extracted.rows>
|
||||
</cfif>
|
||||
|
||||
<!--- Сохранить spec_rows (DELETE + INSERT) --->
|
||||
<cftransaction>
|
||||
<cfquery datasource="baza">
|
||||
DELETE FROM spec_rows WHERE supplement_id = <cfqueryparam value="#sid#" cfsqltype="cf_sql_varchar">
|
||||
</cfquery>
|
||||
<cfloop array="#rows#" index="row">
|
||||
<cfset rn = structKeyExists(row, 'row_num') ? row.row_num : 0>
|
||||
<cfset nm = structKeyExists(row, 'name') ? row.name : ''>
|
||||
<cfset pr = (structKeyExists(row, 'price') AND len(row.price)) ? row.price : ''>
|
||||
<cfset qt = (structKeyExists(row, 'qty') AND len(row.qty)) ? row.qty : ''>
|
||||
<cfset sm = (structKeyExists(row, 'sum') AND len(row.sum)) ? row.sum : ''>
|
||||
<cfset ds = (structKeyExists(row, 'date_start') AND len(row.date_start)) ? row.date_start : ''>
|
||||
|
||||
<cfquery datasource="baza">
|
||||
INSERT INTO spec_rows (supplement_id, row_num, name, price, qty, sum, date_start)
|
||||
VALUES (
|
||||
<cfqueryparam value="#sid#" cfsqltype="cf_sql_varchar">,
|
||||
<cfqueryparam value="#rn#" cfsqltype="cf_sql_integer">,
|
||||
<cfqueryparam value="#nm#" cfsqltype="cf_sql_varchar">,
|
||||
<cfqueryparam value="#pr#" cfsqltype="cf_sql_float" null="#len(pr) EQ 0#">,
|
||||
<cfqueryparam value="#qt#" cfsqltype="cf_sql_float" null="#len(qt) EQ 0#">,
|
||||
<cfqueryparam value="#sm#" cfsqltype="cf_sql_float" null="#len(sm) EQ 0#">,
|
||||
<cfqueryparam value="#ds#" cfsqltype="cf_sql_varchar" null="#len(ds) EQ 0#">
|
||||
)
|
||||
</cfquery>
|
||||
</cfloop>
|
||||
</cftransaction>
|
||||
|
||||
<cfset savedRows = []>
|
||||
<cfloop array="#rows#" index="r">
|
||||
<cfset arrayAppend(savedRows, {ROW_NUM: structKeyExists(r,'row_num')?r.row_num:0, NAME: structKeyExists(r,'name')?r.name:'', PRICE: structKeyExists(r,'price')?r.price:'', QTY: structKeyExists(r,'qty')?r.qty:'', SUM: structKeyExists(r,'sum')?r.sum:'', DATE_START: structKeyExists(r,'date_start')?r.date_start:''})>
|
||||
</cfloop>
|
||||
<cfset extractedRows[sid] = savedRows>
|
||||
<cfset arrayAppend(extractedInfo, {SUPPLEMENT_ID: sid, FILENAME: supps.filename, ROWS_SAVED: arrayLen(savedRows), SKIPPED: false})>
|
||||
|
||||
<cfcatch>
|
||||
<cfset arrayAppend(extractedInfo, {SUPPLEMENT_ID: sid, FILENAME: supps.filename, ERROR: "JSON parse: " & cfcatch.message})>
|
||||
</cfcatch>
|
||||
</cftry>
|
||||
</cfloop>
|
||||
|
||||
<!--- Шаг 2: Differ (логика из differ.cfm, но на свежих данных) --->
|
||||
<cfset allChanges = []>
|
||||
|
||||
<!--- Найти initial --->
|
||||
<cfquery name="initSup" datasource="baza">
|
||||
SELECT id FROM supplements WHERE contract_id=<cfqueryparam value="#url.contract_id#" cfsqltype="cf_sql_varchar"> AND type='initial' ORDER BY created_at LIMIT 1
|
||||
</cfquery>
|
||||
<cfif initSup.recordCount EQ 0>
|
||||
<cfquery name="initSup" datasource="baza">
|
||||
SELECT id FROM supplements WHERE contract_id=<cfqueryparam value="#url.contract_id#" cfsqltype="cf_sql_varchar"> ORDER BY created_at LIMIT 1
|
||||
</cfquery>
|
||||
</cfif>
|
||||
|
||||
<cfif initSup.recordCount GT 0 AND structKeyExists(extractedRows, initSup.id)>
|
||||
<cfset initByNum = {}>
|
||||
<cfloop array="#extractedRows[initSup.id]#" index="r">
|
||||
<cfset initByNum[r.ROW_NUM] = r>
|
||||
</cfloop>
|
||||
|
||||
<cfquery name="amendments" datasource="baza">
|
||||
SELECT id, type FROM supplements WHERE contract_id=<cfqueryparam value="#url.contract_id#" cfsqltype="cf_sql_varchar"> AND id!=<cfqueryparam value="#initSup.id#" cfsqltype="cf_sql_varchar"> ORDER BY created_at
|
||||
</cfquery>
|
||||
|
||||
<cfloop query="amendments">
|
||||
<cfif NOT structKeyExists(extractedRows, amendments.id)>
|
||||
<cfcontinue>
|
||||
</cfif>
|
||||
<cfset amdRows = extractedRows[amendments.id]>
|
||||
<cfset amdByNum = {}>
|
||||
<cfloop array="#amdRows#" index="r">
|
||||
<cfset amdByNum[r.ROW_NUM] = r>
|
||||
</cfloop>
|
||||
|
||||
<cfset allNums = structKeyArray(initByNum)>
|
||||
<cfloop array="#structKeyArray(amdByNum)#" index="n">
|
||||
<cfif NOT structKeyExists(initByNum, n)><cfset arrayAppend(allNums, n)></cfif>
|
||||
</cfloop>
|
||||
<cfset arraySort(allNums, "numeric")>
|
||||
|
||||
<cfset changes = []>
|
||||
<cfset fields = ["NAME","PRICE","QTY","SUM","DATE_START"]>
|
||||
<cfloop array="#allNums#" index="num">
|
||||
<cfset inOld = structKeyExists(initByNum, num)>
|
||||
<cfset inNew = structKeyExists(amdByNum, num)>
|
||||
<cfif inOld AND NOT inNew>
|
||||
<cfset arrayAppend(changes, {ROW_NUM:num, CHANGE_TYPE:"deleted", OLD_VALUES:initByNum[num], NEW_VALUES:{}})>
|
||||
<cfelseif NOT inOld AND inNew>
|
||||
<cfset arrayAppend(changes, {ROW_NUM:num, CHANGE_TYPE:"added", OLD_VALUES:{}, NEW_VALUES:amdByNum[num]})>
|
||||
<cfelse>
|
||||
<cfset ov = initByNum[num]>
|
||||
<cfset nv = amdByNum[num]>
|
||||
<cfset same = true>
|
||||
<cfset cf = {}>
|
||||
<cfloop array="#fields#" index="f">
|
||||
<cfif structKeyExists(ov,f) AND structKeyExists(nv,f)>
|
||||
<cfif toString(ov[f]) NEQ toString(nv[f])>
|
||||
<cfset same = false><cfset cf[f] = {OLD:ov[f], NEW:nv[f]}>
|
||||
</cfif>
|
||||
<cfelseif structKeyExists(ov,f) OR structKeyExists(nv,f)>
|
||||
<cfset same = false><cfset cf[f] = {OLD: structKeyExists(ov,f)?ov[f]:"", NEW: structKeyExists(nv,f)?nv[f]:""}>
|
||||
</cfif>
|
||||
</cfloop>
|
||||
<cfset arrayAppend(changes, {ROW_NUM:num, CHANGE_TYPE:same?"unchanged":"changed", OLD_VALUES:ov, NEW_VALUES:nv, CHANGED_FIELDS:cf})>
|
||||
</cfif>
|
||||
</cfloop>
|
||||
<cfset sm = {ADDED:0, DELETED:0, CHANGED:0, UNCHANGED:0}>
|
||||
<cfloop array="#changes#" index="ch"><cfset sm[ch.CHANGE_TYPE] = sm[ch.CHANGE_TYPE] + 1></cfloop>
|
||||
<cfset arrayAppend(allChanges, {SUPPLEMENT_ID:amendments.id, TYPE:amendments.type, CHANGES:changes, SUMMARY:sm})>
|
||||
</cfloop>
|
||||
</cfif>
|
||||
|
||||
<cfset result = {
|
||||
OK: true,
|
||||
CONTRACT_ID: url.contract_id,
|
||||
EXTRACTED: extractedInfo,
|
||||
ALL_CHANGES: allChanges
|
||||
}>
|
||||
</cfif>
|
||||
</cfif>
|
||||
<cfcatch><cfset result = {OK: false, ERROR: cfcatch.message, DETAIL: cfcatch.detail}></cfcatch>
|
||||
</cftry>
|
||||
<cfoutput>#serializeJSON(result)#</cfoutput>
|
||||
Reference in New Issue
Block a user