smishchuk/payg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

090 s3 from cmdb, inmem DECIMAL fix

Browse Source
This commit is contained in:
smishchuk
2026-10-01 16:59:49 +03:00
parent 669eaec34f
commit e645f0865d
4 changed files with 105 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Application.cfc
+2 -1
View File
@@ -24,6 +24,7 @@
<cfset this.datasources["#this.datasource#"]=getDS("#this.datasource#","ds_#this.datasource#")/> <cfset this.datasources["#this.datasource#"]=getDS("#this.datasource#","ds_#this.datasource#")/>
<cfset this.datasources["billing-vc"]=getDS("billing-vc","ds_billing_vc")/> <cfset this.datasources["billing-vc"]=getDS("billing-vc","ds_billing_vc")/>
<cfset this.datasources["billing-s3"]=getDS("billing-s3","ds_billing_s3")/> <cfset this.datasources["billing-s3"]=getDS("billing-s3","ds_billing_s3")/>
<cfset this.datasources["cmdb"]=getDS("cmdb","ds_cmdb")/>
<!--- но датасорцы таинственно прочухались с прошлым пушем, без переноса инициализации в псевдоконструктор ---> <!--- но датасорцы таинственно прочухались с прошлым пушем, без переноса инициализации в псевдоконструктор --->
<!--- а потом опять... ---> <!--- а потом опять... --->
@@ -88,7 +89,7 @@
<!--- global settings ---> <!--- global settings --->
<!--- *************************************************************************************************** ---> <!--- *************************************************************************************************** --->
<cfset request.APP_VERSION="0.00.089"/> <!--- *********************************************************** ---> <cfset request.APP_VERSION="0.00.090"/> <!--- *********************************************************** --->
<!--- *************************************************************************************************** ---> <!--- *************************************************************************************************** --->
<cfset request.STAND=getStand()/> <cfset request.STAND=getStand()/>
<cfset request.RECORDS_PER_PAGE=500/> <cfset request.RECORDS_PER_PAGE=500/>
MemoryDbHelper.cfc
+2 -1
View File
@@ -1,4 +1,5 @@
component { component {
/*0.04 precision fix DECIMAL(128,32) maybe too memory consumable*/
/*0.03 precision fix*/ /*0.03 precision fix*/
/*0.02 timestamp support*/ /*0.02 timestamp support*/
/** /**
@@ -71,7 +72,7 @@ component {
case "BIGINT": return "BIGINT"; case "BIGINT": return "BIGINT";
case "DOUBLE": case "FLOAT": case "REAL": return "DOUBLE"; case "DOUBLE": case "FLOAT": case "REAL": return "DOUBLE";
case "BOOLEAN": case "BIT": return "BIT"; case "BOOLEAN": case "BIT": return "BIT";
case "DECIMAL": case "NUMERIC": return "DECIMAL"; case "DECIMAL": case "NUMERIC": return "DECIMAL(128,32)";
case "TIMESTAMP": case "DATE": case "TIME": return "TIMESTAMP"; case "TIMESTAMP": case "DATE": case "TIME": return "TIMESTAMP";
default: return "LONGVARCHAR"; // И тут тоже default: return "LONGVARCHAR"; // И тут тоже
} }
etc/notes.txt
+8
View File
@@ -1 +1,9 @@
Задача: S3 перепривязать через CMDB (по именованию аккаунта - не работает (суффиксы, вводимые руками, не гарантированы от ошибок, примеры есть)
Половинчатое решение: мы можем вытащить из CMDB контрагента для S3.
- Сделать источник данных к CMDB (сразу прод?)
- Сделать джойн и либо добавить найденные в CMDB связи с WZ, либо
На самом деле нужно сделать связь инстанса S3 со строкой спецификации.
Контракты и спецификации в CMDB сейчас пустышки, а надо вязать с настоящими контрактами и спецификациями, только тогда возможен правильный биллинг.
Затащить контракты и спецификации в CMDB из CRM? Какие записи отобрать?
12:58 09.10.2025 дополнительное соглашение к сделке. Шедевр 12:58 09.10.2025 дополнительное соглашение к сделке. Шедевр
payg.cfm
+93 -58
View File
@@ -238,7 +238,8 @@ HOT_FREE_LIMIT 1 10 1 100
when bucket_stat.placement_id = 2 then 'paas.s3.cphc' when bucket_stat.placement_id = 2 then 'paas.s3.cphc'
else '['|| TO_CHAR(bucket_stat.placement_id,'FM9') ||']' end as code else '['|| TO_CHAR(bucket_stat.placement_id,'FM9') ||']' end as code
,bucket_stat.placement_id ,bucket_stat.placement_id
,'WZ'||lpad(bucket_stat.owner, 5, '0') as wz -- *** здесь не число, а строка, поэтому другая формула обратной сборки WZ --,'WZ'||lpad(bucket_stat.owner, 5, '0') as wz -- *** здесь не число, а строка, поэтому другая формула обратной сборки WZ
,bucket_stat.owner as s3_user_name
,bucket_info.name as bucket ,bucket_info.name as bucket
from s3billing.bucket_stat from s3billing.bucket_stat
join s3billing.placement ON s3billing.placement.id=bucket_stat.placement_id join s3billing.placement ON s3billing.placement.id=bucket_stat.placement_id
@@ -246,7 +247,7 @@ HOT_FREE_LIMIT 1 10 1 100
where bucket_stat.placement_id in (1,2) where bucket_stat.placement_id in (1,2)
AND bucket_stat.timestamp_addition >= <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dt_start#/> AND bucket_stat.timestamp_addition >= <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dt_start#/>
AND bucket_stat.timestamp_addition < <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dateAdd("d",1,dt_finish)#/> AND bucket_stat.timestamp_addition < <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dateAdd("d",1,dt_finish)#/>
<cfif len(wz)> <cfif len(wz)><!--- *** --->
AND 'WZ'||lpad(bucket_stat.owner, 5, '0') like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#wz#%"/> /*выглядит очень неэффективным запросом*/ AND 'WZ'||lpad(bucket_stat.owner, 5, '0') like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#wz#%"/> /*выглядит очень неэффективным запросом*/
</cfif> </cfif>
group by bucket_stat.owner, bucket_stat.placement_id, bucket_info.name group by bucket_stat.owner, bucket_stat.placement_id, bucket_info.name
@@ -306,8 +307,9 @@ select
else '['|| TO_CHAR(bucket_info.placement_id,'FM9') ||']' end as code --*** зачем делать там ключ bigint else '['|| TO_CHAR(bucket_info.placement_id,'FM9') ||']' end as code --*** зачем делать там ключ bigint
,bucket_info.placement_id ,bucket_info.placement_id
--category.name --category.name
, bucket_info.name as bucket ,bucket_info.name as bucket
,'WZ'||lpad(user_info.name, 5, '0') as wz ,user_info.name as s3_user_name
--,'dummy' AS WZ
from s3billing.usage_bucket_by_user from s3billing.usage_bucket_by_user
join s3billing.user_info on usage_bucket_by_user.user_id=user_info.id join s3billing.user_info on usage_bucket_by_user.user_id=user_info.id
join s3billing.category on usage_bucket_by_user.category_id=category.id join s3billing.category on usage_bucket_by_user.category_id=category.id
@@ -336,7 +338,22 @@ select
<cfoutput>qS3OpsTrfAge:#getTickCount()-request.startTickCount#</cfoutput><cfif !isDefined("output_xls") AND !isDefined("output_json")><cfflush/></cfif> <cfoutput>qS3OpsTrfAge:#getTickCount()-request.startTickCount#</cfoutput><cfif !isDefined("output_xls") AND !isDefined("output_json")><cfflush/></cfif>
<cfquery name="qS3User" datasource="cmdb" cachedWithin=#createTimeSpan(0, 0, 31, 0)#>
WITH current_state AS (
SELECT *, ROW_NUMBER() OVER (PARTITION BY instance_uid ORDER BY version DESC) as rn
FROM instance_state
)
SELECT s.instance_data->'job'->>'contragentCode' as wz, s.instance_data->'job'->>'userId' AS user_id, e.display_name
FROM current_state s
JOIN instance e on (s.instance_uid=e.instance_uid)
WHERE rn = 1 AND e.service_id=12
</cfquery>
<cfoutput>qS3User:#getTickCount()-request.startTickCount#</cfoutput><cfif !isDefined("output_xls") AND !isDefined("output_json")><cfflush/></cfif>
<cfif isDefined("DEBUG")>
<cfdump var=#qS3User#/>
<cfif !isDefined("output_xls") AND !isDefined("output_json")><cfflush/></cfif>
</cfif>
<!--- выданы права на схему и таблицы, чтобы без толку не плодить источники данных ---> <!--- выданы права на схему и таблицы, чтобы без толку не плодить источники данных --->
<!--- считаются часы работы ВМ с GPU ---> <!--- считаются часы работы ВМ с GPU --->
@@ -364,6 +381,9 @@ select
WHERE NOT vm.is_template WHERE NOT vm.is_template
AND p.ts >= <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dt_start#/> AND p.ts >= <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dt_start#/>
AND p.ts < <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dateAdd("d",1,dt_finish)#/> AND p.ts < <cfqueryparam cfsqltype="cf_sql_timestamp" value=#dateAdd("d",1,dt_finish)#/>
<cfif len(wz)>
AND vm.client_id ilike <cfqueryparam cfsqltype="cf_sql_varchar" value="%#wz#%"/>
</cfif>
GROUP BY vm.client_id, item.key, c.code, vm.obj_id GROUP BY vm.client_id, item.key, c.code, vm.obj_id
ORDER BY vm.client_id, item.key, c.code, vm.obj_id; ORDER BY vm.client_id, item.key, c.code, vm.obj_id;
</cfquery> </cfquery>
@@ -381,67 +401,75 @@ select
<cfoutput>qGpuAge:#getTickCount()-request.startTickCount#</cfoutput><cfif !isDefined("output_xls") AND !isDefined("output_json")><cfflush/></cfif> <cfoutput>qGpuAge:#getTickCount()-request.startTickCount#</cfoutput><cfif !isDefined("output_xls") AND !isDefined("output_json")><cfflush/></cfif>
<cfset db = new MemoryDbHelper()/>
<cfset db.register("qComputing", qComputing, "inmem")/>
<cfset db.register("qStorage", qStorage, "inmem")/>
<cfset db.register("qS3Vol", qS3Vol, "inmem")/>
<cfset db.register("qS3OpsTrf", qS3OpsTrf, "inmem")/>
<cfset db.register("qGpu", qGpu, "inmem")/>
<cfset db.register("qS3User", qS3User, "inmem")/>
<cfquery name="qUnifiedMetric" datasource="inmem">
select wz, code || '.vcpu' as code, core_h as raw_metric, core_h as metric, 'core-h' as unit, vdc as dimension1
from qComputing
<cfif len(code)>
where code || '.vcpu' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all
select wz, code || '.ram', gb_h as raw_metric, gb_h as metric, 'GByte-h' as unit, vdc as dimension1
from qComputing
<cfif len(code)>
where code || '.ram' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all
select wz, code, gb_m_used as raw_metric, gb_m_used as metric, 'GByte-m' as unit, vdc ||':'|| profile as dimension1
from qStorage
<cfif len(code)>
where code like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
<cfquery name="qUnifiedMetric" dbType="query"> union all
select wz, code || '.vcpu' as code, core_h as raw_metric, core_h as metric, 'core-h' as unit, vdc as dimension1 select wz, code || '.vol',vol_b as raw_metric, vol_gb as metric, 'GByte-m' as unit, bucket as dimension1
from qComputing from qS3Vol s
<cfif len(code)> left outer join qS3User u ON (s.s3_user_name = u.user_id)
where code || '.vcpu' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/> <cfif len(code)>
</cfif> where code || '.vol' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all union all
select wz, code || '.ram', gb_h as raw_metric, gb_h as metric, 'GByte-h' as unit, vdc as dimension1 select wz, code || '.get', get as raw_metric, get_10k as metric, '10k' as unit, bucket as dimension1
from qComputing from qS3OpsTrf s
<cfif len(code)> left outer join qS3User u ON (s.s3_user_name = u.user_id)
where code || '.ram' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/> <cfif len(code)>
</cfif> where code || '.get' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all union all
select wz, code, gb_m_used as raw_metric, gb_m_used as metric, 'GByte-m' as unit, vdc ||':'|| profile as dimension1 select wz, code || '.put', put as raw_metric, put_10k as metric, '10k' as unit, bucket as dimension1
from qStorage from qS3OpsTrf s
<cfif len(code)> left outer join qS3User u ON (s.s3_user_name = u.user_id)
where code like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/> <cfif len(code)>
</cfif> where code || '.put' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all union all
select wz, code || '.vol', vol_b as raw_metric, vol_gb as metric, 'GByte-m' as unit, bucket as dimension1 select wz, code || '.trf', bytes_sent as raw_metric, bytes_sent_gb as metric, 'GByte' as unit, bucket as dimension1
from qS3Vol from qS3OpsTrf s
<cfif len(code)> left outer join qS3User u ON (s.s3_user_name = u.user_id)
where code || '.vol' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/> <cfif len(code)>
</cfif> where code || '.trf' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all union all
select wz, code || '.get', get as raw_metric, get_10k as metric, '10k' as unit, bucket as dimension1 select wz, code, gpu_h as raw_metric, gpu_h as metric, 'h' as unit, obj_id as dimension1
from qS3OpsTrf from qGpu
<cfif len(code)> <cfif len(code)>
where code || '.get' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/> where code like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif> </cfif>
union all order by wz, code
select wz, code || '.put', put as raw_metric, put_10k as metric, '10k' as unit, bucket as dimension1
from qS3OpsTrf
<cfif len(code)>
where code || '.put' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all
select wz, code || '.trf', bytes_sent as raw_metric, bytes_sent_gb as metric, 'GByte' as unit, bucket as dimension1
from qS3OpsTrf
<cfif len(code)>
where code || '.trf' like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
union all
select wz, code, gpu_h as raw_metric, gpu_h as metric, 'h' as unit, obj_id as dimension1
from qGpu
<cfif len(code)>
where code like <cfqueryparam cfsqltype="cf_sql_varchar" value="%#code#%"/>
</cfif>
order by wz, code
</cfquery> </cfquery>
<!--- <cfdump var=#qSpec#/> <!--- <cfdump var=#qSpec#/>
<cfdump var=#qUnifiedMetric#/> ---> <cfdump var=#qUnifiedMetric#/> --->
@@ -455,7 +483,6 @@ order by wz, code
<!--- загружаем рекордсеты в inmemory db, вместо чтобы мучиться с QoQ ---> <!--- загружаем рекордсеты в inmemory db, вместо чтобы мучиться с QoQ --->
<cfset db = new MemoryDbHelper()/>
<cfset db.register("qUnifiedMetric", qUnifiedMetric, "inmem")/> <cfset db.register("qUnifiedMetric", qUnifiedMetric, "inmem")/>
<cfset db.register("qContragent", qContragent, "inmem")/> <cfset db.register("qContragent", qContragent, "inmem")/>
<cfset db.register("qSpec", qSpec, "inmem")/> <cfset db.register("qSpec", qSpec, "inmem")/>
@@ -474,6 +501,11 @@ FULL JOIN, потому что нам нужны бесхозные записи
Группировка: по контрагенту, Группировка: по контрагенту,
---> --->
<!--- <cfquery name="qSpecTest" datasource="inmem">
select * from qSpec
</cfquery>
<cfdump var=#qSpecTest#/> --->
<cfquery name="qCharge" datasource="inmem"> <cfquery name="qCharge" datasource="inmem">
/*EXPLAIN PLAN FOR*/ /*EXPLAIN PLAN FOR*/
@@ -522,8 +554,10 @@ SELECT
<d:field title="Клиентс. назв.">s.user_description</d:field> <d:field title="Клиентс. назв.">s.user_description</d:field>
<d:field title="Pricing Model">CASE s.type WHEN 1 THEN 'install' WHEN 2 THEN 'fix' WHEN 3 THEN 'payg' ELSE NULL END as pricing_model</d:field> <d:field title="Pricing Model">CASE s.type WHEN 1 THEN 'install' WHEN 2 THEN 'fix' WHEN 3 THEN 'payg' ELSE NULL END as pricing_model</d:field>
</d:field_set> </d:field_set>
,'***' as delim,s.*
FROM qUnifiedMetric m FROM qUnifiedMetric m
<cfif local.inner_join> <cfif local.inner_join>
INNER
<cfelse> <cfelse>
FULL FULL
</cfif> </cfif>
@@ -739,7 +773,7 @@ S3 операции и трафик: <b<cfif datediff("h",qS3OpsTrfAge.dt_load,
<tr> <tr>
<td colspan="4"></td> <td colspan="4"></td>
<td colspan="2" class="b r">Итого #code#:</td> <td colspan="2" class="b r">Итого #code#:</td>
<td class="b r" style="font-size:110%; padding:0 1em;">#accum_metric_dimension_1#<cfif final_metric_dimension_1 NEQ 0 AND isNumeric(final_metric_dimension_1)>#round(final_metric_dimension_1,3)#</cfif></td> <td class="b r" style="font-size:110%; padding:0 1em;"><!--- #accum_metric_dimension_1# ---><cfif final_metric_dimension_1 NEQ 0 AND isNumeric(final_metric_dimension_1)>#round(final_metric_dimension_1,3)#</cfif></td>
<td class="b r">#free_tier#</td> <td class="b r">#free_tier#</td>
<td class="b r" style="font-size:120%; padding:0 1em;"><cfif final_metric_dimension_1 NEQ 0 AND isNumeric(final_metric_dimension_1)>#max(ceiling(final_metric_dimension_1) - free_tier, 0)#</cfif></td> <td class="b r" style="font-size:120%; padding:0 1em;"><cfif final_metric_dimension_1 NEQ 0 AND isNumeric(final_metric_dimension_1)>#max(ceiling(final_metric_dimension_1) - free_tier, 0)#</cfif></td>
<td class="b c">#unit# <cfif len(ed_izm)>[#ed_izm#]</cfif></td> <td class="b c">#unit# <cfif len(ed_izm)>[#ed_izm#]</cfif></td>
@@ -768,6 +802,7 @@ S3 операции и трафик: <b<cfif datediff("h",qS3OpsTrfAge.dt_load,
<cfdump var=#this.datasources[request.ds].connectionString#/> <cfdump var=#this.datasources[request.ds].connectionString#/>
<cfdump var=#this.datasources["billing-vc"].connectionString#/> <cfdump var=#this.datasources["billing-vc"].connectionString#/>
<cfdump var=#this.datasources["billing-s3"].connectionString#/> <cfdump var=#this.datasources["billing-s3"].connectionString#/>
<cfdump var=#this.datasources["cmdb"].connectionString#/>
<layout:page section="footer"/> <layout:page section="footer"/>