021 dimensions dimension1

This commit is contained in:
2026-07-20 11:06:58 +03:00
parent 90aa268918
commit 3bc7cea1d1
16 changed files with 2121 additions and 185 deletions
+205
View File
@@ -0,0 +1,205 @@
DROP FOREIGN TABLE IF EXISTS vmreports.compute CASCADE;
DROP FOREIGN TABLE IF EXISTS vmreports.storage CASCADE;
DROP FOREIGN TABLE IF EXISTS vmreports.host CASCADE;
DROP FOREIGN TABLE IF EXISTS vmreports.vm_gpu CASCADE;
DROP FOREIGN TABLE IF EXISTS vmreports.vm_power_state CASCADE;
--------------------------------------------------------------------------------
CREATE SCHEMA IF NOT EXISTS vmreports;
ALTER SCHEMA vmreports OWNER TO dbo;
GRANT USAGE ON SCHEMA vmreports TO nifi,appserver,reader,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO appserver,reader;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT, REFERENCES ON TABLES TO appserver,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT ALL ON TABLES TO nifi;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO nifi, appserver;
SET default_tablespace = '';
SET default_table_access_method = heap;
--
-- TOC entry 721 (class 1259 OID 9816992)
-- Name: compute; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.compute (
"Name" character varying(255),
"UUID_VM" character varying(255),
"VDC" character varying(255),
"Organization_UUID" uuid,
"Organization" character varying(255),
"WZ" character varying(255),
"Cluster" character varying(255),
"Host" character varying(255),
"Host_Model" character varying(255),
"CPU_Model" character varying(255),
"Host_CPU_Cores" double precision,
"Host_CPU_Sockets" double precision,
"Host_CPU_Speed" double precision,
"Host_RAM" double precision,
"Host_CPU_Used" double precision,
"Host_CPU_Used_Avg" double precision,
"VDC_CPU_Reserv" double precision,
"VDC_CPU_Speed" double precision,
"VM_vCPU_count" double precision,
"VM_CPU_Used" double precision,
"VM_CPU_Used_AVG" double precision,
"VM_CPU_Reserv_GHz" double precision,
"VM_CPU_Reserv_perc" double precision,
"VM_RAM_Allocated" double precision,
"VM_RAM_Used" double precision,
"VM_is_ON" double precision,
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL
);
ALTER TABLE vmreports.compute OWNER TO dbo;
--
-- TOC entry 722 (class 1259 OID 9817004)
-- Name: storage; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.storage (
"Name" character varying(255),
"VM_UUID" character varying(255),
"VDC" character varying(255),
"Organization_UUID" uuid,
"Organization" character varying(255),
"WZ" character varying(255),
"Cluster" character varying(255),
"Disk" character varying(255),
"Storage_Policy" character varying(255),
"Disk_Allocated" double precision,
"Disk_Used" double precision,
"VM_is_ON" double precision,
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL
);
ALTER TABLE vmreports.storage OWNER TO dbo;
--
-- TOC entry 720 (class 1259 OID 9816984)
-- Name: host; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.host (
"Name" character varying(255),
"Model" character varying(255),
"Cluster" character varying(255),
"Maintenance" character varying(255),
"CPU_Cores" double precision,
"CPU_Sockets" double precision,
"CPU_Speed" double precision,
"RAM_Total" double precision,
"RAM_Used" double precision,
"CPU_Used" double precision,
"CPU_Used_AVG" double precision,
"CPU_Contention_95" double precision,
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL,
"Cluster_v2" character varying(255),
"HT_Active" boolean,
"CPUtoVM" double precision,
"RAMtoVM" double precision
);
ALTER TABLE vmreports.host OWNER TO dbo;
--
-- TOC entry 733 (class 1259 OID 11093092)
-- Name: vm_gpu; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.vm_gpu (
"Name" character varying(255),
"ClientID" character varying(255),
"VCD" character varying(255),
"Org" character varying(255),
"OrgVDC" character varying(255),
"vCenter" character varying(255),
"GPU" jsonb,
"OrgStatus" character varying(255),
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL
);
ALTER TABLE vmreports.vm_gpu OWNER TO dbo;
--
-- TOC entry 732 (class 1259 OID 11093087)
-- Name: vm_power_state; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.vm_power_state (
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL,
value double precision
);
ALTER TABLE vmreports.vm_power_state OWNER TO dbo;
--
-- TOC entry 4761 (class 2606 OID 9816998)
-- Name: compute pk_compute; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.compute
ADD CONSTRAINT pk_compute PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4759 (class 2606 OID 9816990)
-- Name: host pk_host; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.host
ADD CONSTRAINT pk_host PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4763 (class 2606 OID 9817010)
-- Name: storage pk_storage; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.storage
ADD CONSTRAINT pk_storage PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4767 (class 2606 OID 11093098)
-- Name: vm_gpu pk_vm_gpu; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.vm_gpu
ADD CONSTRAINT pk_vm_gpu PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4765 (class 2606 OID 11093091)
-- Name: vm_power_state pk_vm_power_state; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.vm_power_state
ADD CONSTRAINT pk_vm_power_state PRIMARY KEY ("Object_UUID", ts);
--
-- TOC entry 5168 (class 0 OID 0)
-- Dependencies: 10
-- Name: SCHEMA vmreports; Type: ACL; Schema: -; Owner: dbo
--
GRANT ALL ON SCHEMA vmreports TO nifi;
GRANT ALL ON ALL TABLES IN SCHEMA vmreports TO nifi;
GRANT SELECT, REFERENCES ON ALL TABLES IN SCHEMA vmreports TO analytics, analyst;
+128
View File
@@ -0,0 +1,128 @@
CREATE SCHEMA IF NOT EXISTS vmreports;
ALTER SCHEMA vmreports OWNER TO dbo;
GRANT USAGE ON SCHEMA vmreports TO nifi;
GRANT ALL ON ALL TABLES IN SCHEMA vmreports TO nifi;
GRANT ALL ON ALL SEQUENCES IN SCHEMA vmreports TO nifi;
--ALTER DEFAULT PRIVILEGES IN SCHEMA vmreports GRANT ALL ON TABLES TO nifi;
--ALTER DEFAULT PRIVILEGES IN SCHEMA vmreports GRANT ALL ON SEQUENCES TO nifi;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo, postgres IN SCHEMA vmreports GRANT ALL ON TABLES TO nifi;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo, postgres IN SCHEMA vmreports GRANT SELECT ON TABLES TO appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo, postgres IN SCHEMA vmreports GRANT ALL ON SEQUENCES TO nifi;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo, postgres IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO appserver;
drop table if exists vmreports.host;
create table vmreports.host(
"Name" varchar(255) NULL,
"Model" varchar(255) NULL,
"Cluster" varchar(255) NULL,
"Cluster_v2" varchar(255) NULL,
"Maintenance" varchar(255) NULL,
"CPU_Cores" float NULL, -- per host, not per socket
"CPU_Sockets" float NULL, -- not used in caclulations
"CPU_Speed" float NULL,
"RAM_Total" float NULL,
"RAM_Used" float NULL,
"CPU_Used" float NULL,
"CPU_Used_AVG" float NULL,
"CPU_Contention_95" float NULL,
"HT_Active" boolean NULL,
"CPUtoVM" float NULL,
"RAMtoVM" float NULL,
"Object_UUID" uuid NOT NULL,
ts timestamptz NOT NULL
,CONSTRAINT PK_host PRIMARY KEY (ts, "Object_UUID")
);
--grant all on vmreports.host to nifi;
drop table if exists vmreports.compute;
create table vmreports.compute (
"Name" varchar(255) NULL,
"UUID_VM" varchar(255) NULL,
"VDC" varchar(255) NULL,
"Organization_UUID" uuid NULL,
"Organization" varchar(255) NULL,
"WZ" varchar(255) NULL,
"Cluster" varchar(255) NULL,
"Host" varchar(255) NULL,
"Host_Model" varchar(255) NULL,
"CPU_Model" varchar(255) NULL,
"Host_CPU_Cores" float NULL,
"Host_CPU_Sockets" float NULL,
"Host_CPU_Speed" float NULL,
"Host_RAM" float NULL,
"Host_CPU_Used" float NULL,
"Host_CPU_Used_Avg" float NULL,
"VDC_CPU_Reserv" float NULL,
"VDC_CPU_Speed" float NULL,
"VM_vCPU_count" float NULL,
"VM_CPU_Used" float NULL,
"VM_CPU_Used_AVG" float NULL,
"VM_CPU_Reserv_GHz" float NULL,
"VM_CPU_Reserv_perc" float NULL,
"VM_RAM_Allocated" float NULL,
"VM_RAM_Used" float NULL,
"VM_is_ON" float NULL,
--"Summary": false,
--"Grand Total": false,
--"Group UUID": null,
"Object_UUID" uuid NOT NULL,
ts timestamptz NOT NULL
,CONSTRAINT PK_compute PRIMARY KEY (ts, "Object_UUID")
);
--grant all on vmreports.compute to nifi;
drop table if exists vmreports.storage;
create table vmreports.storage (
"Name" varchar(255) NULL,
"VM_UUID" varchar(255) NULL,
"VDC" varchar(255) NULL,
"Organization_UUID" uuid NULL,
"Organization" varchar(255) NULL,
"WZ" varchar(255) NULL,
"Cluster" varchar(255) NULL,
"Disk" varchar(255) NULL,
"Storage_Policy" varchar(255) NULL,
"Disk_Allocated" float NULL,
"Disk_Used" float NULL,
"VM_is_ON" float NULL,
"Object_UUID" uuid NOT NULL,
ts timestamptz NOT NULL
,CONSTRAINT PK_storage PRIMARY KEY (ts, "Object_UUID")
);
--grant all on vmreports.storage to nifi;
drop table if exists vmreports.vm_gpu;
create table vmreports.vm_gpu (
"Name" varchar(255) NULL,
"ClientID" varchar(255) NULL, --WZ
"VCD" varchar(255) NULL, --vcloud
"Org" varchar(255) NULL,
"OrgVDC" varchar(255) NULL,
"vCenter" varchar(255) NULL,
"GPU" jsonb NULL,
"OrgStatus" varchar(255) NULL,
"Object_UUID" uuid NOT NULL,
ts timestamptz NOT NULL
,CONSTRAINT PK_vm_gpu PRIMARY KEY (ts, "Object_UUID")
);
--grant all on vmreports.vm_gpu to nifi;
DROP TABLE IF EXISTS vmreports.vm_power_state;
CREATE TABLE IF NOT EXISTS vmreports.vm_power_state
(
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL,
value float,
CONSTRAINT PK_vm_power_state PRIMARY KEY ("Object_UUID", ts)
);
--grant all on vmreports.vm_power_state to nifi;
+204
View File
@@ -0,0 +1,204 @@
--
-- PostgreSQL database dump
--
--
-- TOC entry 10 (class 2615 OID 9816969)
-- Name: vmreports; Type: SCHEMA; Schema: -; Owner: dbo
--
CREATE SCHEMA IF NOT EXISTS vmreports;
ALTER SCHEMA vmreports OWNER TO dbo;
GRANT USAGE ON SCHEMA vmreports TO nifi,appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT ALL ON TABLES TO nifi;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO nifi, appserver;
SET default_tablespace = '';
SET default_table_access_method = heap;
--
-- TOC entry 721 (class 1259 OID 9816992)
-- Name: compute; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.compute (
"Name" character varying(255),
"UUID_VM" character varying(255),
"VDC" character varying(255),
"Organization_UUID" uuid,
"Organization" character varying(255),
"WZ" character varying(255),
"Cluster" character varying(255),
"Host" character varying(255),
"Host_Model" character varying(255),
"CPU_Model" character varying(255),
"Host_CPU_Cores" double precision,
"Host_CPU_Sockets" double precision,
"Host_CPU_Speed" double precision,
"Host_RAM" double precision,
"Host_CPU_Used" double precision,
"Host_CPU_Used_Avg" double precision,
"VDC_CPU_Reserv" double precision,
"VDC_CPU_Speed" double precision,
"VM_vCPU_count" double precision,
"VM_CPU_Used" double precision,
"VM_CPU_Used_AVG" double precision,
"VM_CPU_Reserv_GHz" double precision,
"VM_CPU_Reserv_perc" double precision,
"VM_RAM_Allocated" double precision,
"VM_RAM_Used" double precision,
"VM_is_ON" double precision,
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL
);
ALTER TABLE vmreports.compute OWNER TO dbo;
--
-- TOC entry 722 (class 1259 OID 9817004)
-- Name: storage; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.storage (
"Name" character varying(255),
"VM_UUID" character varying(255),
"VDC" character varying(255),
"Organization_UUID" uuid,
"Organization" character varying(255),
"WZ" character varying(255),
"Cluster" character varying(255),
"Disk" character varying(255),
"Storage_Policy" character varying(255),
"Disk_Allocated" double precision,
"Disk_Used" double precision,
"VM_is_ON" double precision,
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL
);
ALTER TABLE vmreports.storage OWNER TO dbo;
--
-- TOC entry 720 (class 1259 OID 9816984)
-- Name: host; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.host (
"Name" character varying(255),
"Model" character varying(255),
"Cluster" character varying(255),
"Maintenance" character varying(255),
"CPU_Cores" double precision,
"CPU_Sockets" double precision,
"CPU_Speed" double precision,
"RAM_Total" double precision,
"RAM_Used" double precision,
"CPU_Used" double precision,
"CPU_Used_AVG" double precision,
"CPU_Contention_95" double precision,
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL,
"Cluster_v2" character varying(255),
"HT_Active" boolean,
"CPUtoVM" double precision,
"RAMtoVM" double precision
);
ALTER TABLE vmreports.host OWNER TO dbo;
--
-- TOC entry 733 (class 1259 OID 11093092)
-- Name: vm_gpu; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.vm_gpu (
"Name" character varying(255),
"ClientID" character varying(255),
"VCD" character varying(255),
"Org" character varying(255),
"OrgVDC" character varying(255),
"vCenter" character varying(255),
"GPU" jsonb,
"OrgStatus" character varying(255),
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL
);
ALTER TABLE vmreports.vm_gpu OWNER TO dbo;
--
-- TOC entry 732 (class 1259 OID 11093087)
-- Name: vm_power_state; Type: TABLE; Schema: vmreports; Owner: smishchuk
--
CREATE TABLE vmreports.vm_power_state (
"Object_UUID" uuid NOT NULL,
ts timestamp with time zone NOT NULL,
value double precision
);
ALTER TABLE vmreports.vm_power_state OWNER TO dbo;
--
-- TOC entry 4761 (class 2606 OID 9816998)
-- Name: compute pk_compute; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.compute
ADD CONSTRAINT pk_compute PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4759 (class 2606 OID 9816990)
-- Name: host pk_host; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.host
ADD CONSTRAINT pk_host PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4763 (class 2606 OID 9817010)
-- Name: storage pk_storage; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.storage
ADD CONSTRAINT pk_storage PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4767 (class 2606 OID 11093098)
-- Name: vm_gpu pk_vm_gpu; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.vm_gpu
ADD CONSTRAINT pk_vm_gpu PRIMARY KEY (ts, "Object_UUID");
--
-- TOC entry 4765 (class 2606 OID 11093091)
-- Name: vm_power_state pk_vm_power_state; Type: CONSTRAINT; Schema: vmreports; Owner: smishchuk
--
ALTER TABLE ONLY vmreports.vm_power_state
ADD CONSTRAINT pk_vm_power_state PRIMARY KEY ("Object_UUID", ts);
--
-- TOC entry 5168 (class 0 OID 0)
-- Dependencies: 10
-- Name: SCHEMA vmreports; Type: ACL; Schema: -; Owner: dbo
--
GRANT ALL ON SCHEMA vmreports TO nifi;
+101
View File
@@ -0,0 +1,101 @@
ALTER TABLE IF EXISTS vmreports.storage
OWNER to dbo;
REVOKE ALL ON TABLE vmreports.storage FROM analyst;
REVOKE ALL ON TABLE vmreports.storage FROM analytics;
REVOKE ALL ON TABLE vmreports.storage FROM appserver;
GRANT SELECT ON TABLE vmreports.storage TO analyst;
GRANT SELECT ON TABLE vmreports.storage TO analytics;
GRANT SELECT ON TABLE vmreports.storage TO appserver;
GRANT ALL ON TABLE vmreports.storage TO dbo;
GRANT ALL ON TABLE vmreports.storage TO designer;
GRANT ALL ON TABLE vmreports.storage TO nifi;
---------------
ALTER TABLE IF EXISTS vmreports.compute
OWNER to dbo;
REVOKE ALL ON TABLE vmreports.compute FROM analyst;
REVOKE ALL ON TABLE vmreports.compute FROM analytics;
REVOKE ALL ON TABLE vmreports.compute FROM appserver;
GRANT SELECT ON TABLE vmreports.compute TO analyst;
GRANT SELECT ON TABLE vmreports.compute TO analytics;
GRANT SELECT ON TABLE vmreports.compute TO appserver;
GRANT ALL ON TABLE vmreports.compute TO dbo;
GRANT ALL ON TABLE vmreports.compute TO designer;
GRANT ALL ON TABLE vmreports.compute TO nifi;
--------------------
ALTER TABLE IF EXISTS vmreports.host
OWNER to dbo;
REVOKE ALL ON TABLE vmreports.host FROM analyst;
REVOKE ALL ON TABLE vmreports.host FROM analytics;
REVOKE ALL ON TABLE vmreports.host FROM appserver;
GRANT SELECT ON TABLE vmreports.host TO analyst;
GRANT SELECT ON TABLE vmreports.host TO analytics;
GRANT SELECT ON TABLE vmreports.host TO appserver;
GRANT ALL ON TABLE vmreports.host TO dbo;
GRANT ALL ON TABLE vmreports.host TO designer;
GRANT ALL ON TABLE vmreports.host TO nifi;
------------------------------------
ALTER TABLE IF EXISTS vmreports.vm_gpu
OWNER to dbo;
REVOKE ALL ON TABLE vmreports.vm_gpu FROM analyst;
REVOKE ALL ON TABLE vmreports.vm_gpu FROM analytics;
REVOKE ALL ON TABLE vmreports.vm_gpu FROM appserver;
GRANT SELECT ON TABLE vmreports.vm_gpu TO analyst;
GRANT SELECT ON TABLE vmreports.vm_gpu TO analytics;
GRANT SELECT ON TABLE vmreports.vm_gpu TO appserver;
GRANT ALL ON TABLE vmreports.vm_gpu TO dbo;
GRANT ALL ON TABLE vmreports.vm_gpu TO designer;
GRANT ALL ON TABLE vmreports.vm_gpu TO nifi;
-----------------
ALTER TABLE IF EXISTS vmreports.vm_power_state
OWNER to dbo;
REVOKE ALL ON TABLE vmreports.vm_power_state FROM analyst;
REVOKE ALL ON TABLE vmreports.vm_power_state FROM analytics;
REVOKE ALL ON TABLE vmreports.vm_power_state FROM appserver;
GRANT SELECT ON TABLE vmreports.vm_power_state TO analyst;
GRANT SELECT ON TABLE vmreports.vm_power_state TO analytics;
GRANT SELECT ON TABLE vmreports.vm_power_state TO appserver;
GRANT ALL ON TABLE vmreports.vm_power_state TO dbo;
GRANT ALL ON TABLE vmreports.vm_power_state TO designer;
GRANT ALL ON TABLE vmreports.vm_power_state TO nifi;
+65
View File
@@ -0,0 +1,65 @@
SET ROLE dbo;
DROP SCHEMA IF EXISTS vmreports CASCADE;
CREATE SCHEMA IF NOT EXISTS vmreports AUTHORIZATION dbo;
GRANT USAGE ON SCHEMA vmreports TO appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO appserver;
GRANT USAGE ON SCHEMA vmreports TO nifi,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO nifi,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO nifi,analytics,analyst;
/*
конструкция осложняется тем, что
нужен один пользователь-владелец и второй читатель
Это вызывает множество телодвижений при создании.
Необходимость указывать пароль при локальном коннекте не очень радует
(ведь пользователь локальный, прописан на этом же сервере)
*/
--DROP FOREIGN TABLE IF EXISTS host;
--CREATE FOREIGN TABLE vmreports.host () SERVER billing OPTIONS (schema_name 'vmreports', table_name 'host'); -- вместо этого лучше импортировать схему
DROP SERVER IF EXISTS billing CASCADE;
CREATE SERVER billing
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'localhost', port '5432', dbname 'billing', updatable 'false');
-- для импорта схемы мы создаем маппинг владельцу
DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER billing;
CREATE USER MAPPING FOR PUBLIC SERVER billing OPTIONS (user 'appserver', password '***');
IMPORT FOREIGN SCHEMA vmreports FROM SERVER billing INTO vmreports;
--GRANT SELECT ON vmreports.host to appserver;
--DROP USER MAPPING IF EXISTS FOR dbo SERVER billing; -- мы импортировали схему, маппинг больше не нужен.
-- Но нам все равно пришлось указать пароль дважды
--DROP USER MAPPING IF EXISTS FOR appserver SERVER billing;
--CREATE USER MAPPING FOR appserver SERVER billing OPTIONS (user 'nifi', password '***');
--проверка
--set role appserver; select * from vmreports.host;
-- это если интересно, не разрешен ли локальный коннект
--SELECT * FROM pg_hba_file_rules;
/*
Довольно тонкий нюанс, легко ускользающий
ALTER DEFAULT PRIVILEGES FOR ROLE target_creator_role
IN SCHEMA schema_name
GRANT privilege_type ON TABLES TO recipient_role;
Recommended Strategy for "Any User"Because there is no "FOR ALL ROLES" option, the best practice is to have all users create objects as a shared group role.Create a group role (e.g., schema_owner_role).Grant that role to all relevant users.Instruct users to run SET ROLE schema_owner_role; before creating objects.Run ALTER DEFAULT PRIVILEGES FOR ROLE schema_owner_role ... once to cover all objects created under that role.
*/
/*
Best Practices for Role Owners:Dedicated Schema Owner: Create a dedicated, non-login role for managing database schema objects (DDL) and make it the owner of the database/schemas.Separation of Duties: Separate the application user (DML - SELECT, INSERT, UPDATE) from the owner role (DDL - CREATE, ALTER, DROP).Use ALTER DEFAULT PRIVILEGES: Set up default privileges for new objects to ensure that user roles automatically gain access to new tables or functions created by the owner role.Leverage Role Hierarchy: Create group roles (e.g., app_reader, app_writer) and grant these to individual user roles for easier permission management.Avoid Public Schema: Create specific schemas and revoke CREATE privileges on the public schema to prevent accidental object creation.Secure SUPERUSER Usage: Only use the default postgres superuser role for initial setup and critical administrative tasks. Never use it for applications.Example Implementation:sql-- Create a schema owner
CREATE ROLE ddl_user NOLOGIN;
-- Create an app user
CREATE ROLE dml_user LOGIN PASSWORD 'secure_password';
-- Assign ownership
ALTER DATABASE mydb OWNER TO ddl_user;
ALTER SCHEMA public OWNER TO ddl_user;
-- Grant future access
ALTER DEFAULT PRIVILEGES FOR ROLE ddl_user IN SCHEMA public
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO dml_user;
*/
+37
View File
@@ -0,0 +1,37 @@
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO dbo;
SET ROLE dbo;
DROP SCHEMA IF EXISTS vmreports CASCADE;
CREATE SCHEMA IF NOT EXISTS vmreports AUTHORIZATION dbo;
GRANT USAGE ON SCHEMA vmreports TO nifi,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO nifi,analytics,analyst; -- это работает и для внешних таблиц
--ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO nifi,analytics,analyst;
/*
конструкция осложняется тем, что
нужен один пользователь-владелец и второй читатель
Это вызывает множество телодвижений при создании.
Необходимость указывать пароль при локальном коннекте не очень радует
(ведь пользователь локальный, прописан на этом же сервере)
*/
DROP SERVER IF EXISTS billing CASCADE;
CREATE SERVER billing
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'localhost', port '5432', dbname 'billing', updatable 'false');
DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER billing;
CREATE USER MAPPING FOR PUBLIC SERVER billing OPTIONS (user 'nifi', password '***');
IMPORT FOREIGN SCHEMA vmreports FROM SERVER billing INTO vmreports;
--select * from vmreports.host;
/*
план миграции
Забекапить схему vmreports
Восстановить схему vmreports в базу биллинг
Поменять целевую базу импорта НиФи (для прода - втащить новый обмен из репозитория)
Выполнить скрипт в базе DWH - он удалит физические таблицы и создаст внешние таблицы на базу billing
*/
+113
View File
@@ -0,0 +1,113 @@
-- выполнить в базе dwh для создания внешних таблиц схемы vmreports
-- перед выполнением забекапить схему vmreports для восстановления в БД billing
CREATE EXTENSION IF NOT EXISTS postgres_fdw;
GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO dbo;
SET ROLE dbo;
--DROP SCHEMA IF EXISTS vmreports CASCADE;
CREATE SCHEMA IF NOT EXISTS vmreports AUTHORIZATION dbo;
GRANT USAGE ON SCHEMA vmreports TO nifi,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO nifi,analytics,analyst; -- это работает и для внешних таблиц
--ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO nifi,analytics,analyst;
/*
конструкция осложняется тем, что
нужен один пользователь-владелец и второй читатель
Это вызывает множество телодвижений при создании.
Необходимость указывать пароль при локальном коннекте не очень радует
(ведь пользователь локальный, прописан на этом же сервере)
А при смене его пароля все сломается, и поиск ошибки будет неочевиден
*/
DO $$
DECLARE
rec RECORD;
BEGIN
FOR rec IN
SELECT tablename
FROM pg_tables
WHERE schemaname = 'vmreports'
LOOP
EXECUTE format('ALTER TABLE vmreports.%I RENAME TO %I',
rec.tablename,
'del_' || rec.tablename);
END LOOP;
END $$;
DROP SERVER IF EXISTS billing CASCADE;
CREATE SERVER billing
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'localhost', port '5432', dbname 'billing', updatable 'false');
DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER billing;
CREATE USER MAPPING FOR PUBLIC SERVER billing OPTIONS (user 'nifi', password '***');
IMPORT FOREIGN SCHEMA vmreports FROM SERVER billing INTO vmreports;
--select * from vmreports.host;
/*
план миграции - черновик
Забекапить схему vmreports
Восстановить схему vmreports из бэкапа в базу биллинг
Поменять целевую базу импорта НиФи (для прода - втащить новый обмен из репозитория)
Выполнить скрипт в базе DWH - он удалит (переименует) физические таблицы и создаст внешние таблицы на базу billing
*/
/*
Исходно данные host, compute, storage, (gpu)
загружались в dwh.vmreports,
для анализа capacity
В текущей конфигурации,
чтобы избежать дублирования
таблиц, данных и обменов NiFi
Направляем загрузку данных из VROPS в схему billing.vmreports
и в базе dwh (схема vmreports) создаем Foreign Tables на billing.vmreports.
-----------------------------------------------
Миграция на проде:
убедиться, что новый обмен VROPS->billing.vmreports (billing/vmReports) загружен в NiFi Registry
найти пароль nifi (под ним коннект к localhost)
остановить загрузку из VROPS на nifi-01 (billing/vmReports),
сделать бэкап схемы dwh.vmreports (займет минут 10 ... заняло 215 сек),
Восстановить схему vmreports в БД billing - запустить восстановление (Clean Before Restore) (РИСК: восстановление может занять много времени и завершиться с ошибками, обычно так бывает)
select count (*) from vmreports.storage
Выполнить этот скрипт (заменив пароль nifi на актуальный перед выполнением)
переименовать таблицы из схемы vmreports для удаления (есть в скрипте выше)
создать Foreign Tables (коннект проверится при импорте схемы)
Проверить, что права назначились коррректно
Обновить обмен billing/vmReports на продуктиве, запустить
-----------------------------------------------------------------
Данные из billing.vmreports также реплицируются
(процессорами группы LK)
в буферную базу rpt
*/
/*
17:04:49 MSKERROR0dff1f93-153d-3a90-ed64-55d23960b88c
EvaluateJsonPath[id=0dff1f93-153d-3a90-ed64-55d23960b88c] FlowFile FlowFile[filename=06c68bec-d824-4114-913b-bda34cb4fb1d] did not have valid JSON content.
-- со второй попытки отработало
17:05:15 MSKERROR239db407-76f2-3660-7fad-c4bedff6a4f8
InvokeHTTP[id=239db407-76f2-3660-7fad-c4bedff6a4f8] Request Processing failed: FlowFile[filename=3474c340-f360-4a1e-a27e-283be41fc264]: java.net.SocketTimeoutException: timeout
- Caused by: java.net.SocketException: Socket closed
https://payg.adl.nubes.ru/payg.cfm с первого раза таймаут, со второго посчиталось
*/
/*
теперь ищу в нифи примеры с кроном
17:25:05 MSKERROR16efa0f6-f1e7-3755-04fc-5be0383244ec
PutDatabaseRecord[id=16efa0f6-f1e7-3755-04fc-5be0383244ec] Failed to put Records to database for FlowFile[filename=11f8176f-2a98-4199-932e-22e2fc2073b1]. Routing to failure.: org.apache.nifi.serialization.MalformedRecordException: Record does not have a value for the UpdateKey column 'obj_uuid'
17:25:05 MSKERROR16efa0f6-f1e7-3755-04fc-5be0383244ec
PutDatabaseRecord[id=16efa0f6-f1e7-3755-04fc-5be0383244ec] Record does not have a value for the UpdateKey column 'obj_uuid'
17:25:05 MSKERROR16efa0f6-f1e7-3755-04fc-5be0383244ec
PutDatabaseRecord[id=16efa0f6-f1e7-3755-04fc-5be0383244ec] Failed to put Records to database for FlowFile[filename=5039a3b6-2244-4451-81fc-deb8074f3cc7]. Routing to failure.: org.apache.nifi.serialization.MalformedRecordException: Record does not have a value for the UpdateKey column 'obj_uuid'
17:25:05 MSKERROR16efa0f6-f1e7-3755-04fc-5be0383244ec
PutDatabaseRecord[id=16efa0f6-f1e7-3755-04fc-5be0383244ec] Record does not have a value for the UpdateKey column 'obj_uuid'
17:25:05 MSKERROR16efa0f6-f1e7-3755-04fc-5be0383244ec
PutDatabaseRecord[id=16efa0f6-f1e7-3755-04fc-5be0383244ec] Failed to put Records to database for FlowFile[filename=8c6e9efb-198d-4303-897a-53fcfdf4d0f6]. Routing to failure.: org.apache.nifi.serialization.MalformedRecordException: Record does not have a value for the UpdateKey column 'obj_uuid'
*/
+47
View File
@@ -0,0 +1,47 @@
SET ROLE dbo;
DROP SCHEMA IF EXISTS vmreports CASCADE;
CREATE SCHEMA IF NOT EXISTS vmreports AUTHORIZATION dbo;
--GRANT ALL ON SCHEMA vmreports TO postgres;
BEGIN
GRANT USAGE ON SCHEMA vmreports TO appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO appserver;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO appserver;
EXCEPTION END;
BEGIN
GRANT USAGE ON SCHEMA vmreports TO nifi,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT SELECT ON TABLES TO nifi,analytics,analyst;
ALTER DEFAULT PRIVILEGES FOR ROLE dbo IN SCHEMA vmreports GRANT USAGE ON SEQUENCES TO nifi,analytics,analyst;
EXCEPTION WHEN others THEN END;
/*
конструкция осложняется тем, что
нужен один пользователь-владелец и второй читатель
Это вызывает множество телодвижений при создании.
Необходимость указывать пароль при локальном коннекте не очень радует
(ведь пользователь локальный, прописан на этом же сервере)
*/
--DROP FOREIGN TABLE IF EXISTS host;
--CREATE FOREIGN TABLE vmreports.host () SERVER billing OPTIONS (schema_name 'vmreports', table_name 'host'); -- вместо этого лучше импортировать схему
DROP SERVER IF EXISTS billing CASCADE;
CREATE SERVER billing
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'localhost', port '5432', dbname 'billing');
-- для импорта схемы мы создаем маппинг владельцу
DROP USER MAPPING IF EXISTS FOR dbo SERVER billing;
CREATE USER MAPPING FOR dbo SERVER billing OPTIONS (user 'appserver', password '***');
IMPORT FOREIGN SCHEMA vmreports FROM SERVER billing INTO vmreports;
GRANT SELECT ON vmreports.host to appserver;
DROP USER MAPPING IF EXISTS FOR dbo SERVER billing; -- мы импортировали схему, маппинг больше не нужен.
-- Но нам все равно пришлось указать пароль дважды
DROP USER MAPPING IF EXISTS FOR appserver SERVER billing;
CREATE USER MAPPING FOR appserver SERVER billing OPTIONS (user 'appserver', password '***');
--проверка
--set role appserver; select * from vmreports.host;
-- это если интересно, не разрешен ли локальный коннект
--SELECT * FROM pg_hba_file_rules;
+52
View File
@@ -0,0 +1,52 @@
SET ROLE dbo;
DROP TABLE IF EXISTS vmreports.vsan_storage CASCADE;
CREATE TABLE vmreports.vsan_storage (
"Name" varchar(255) NULL,
"ClusterNew" varchar(255) NULL,
"Total" float NULL,
"Usable" float NULL,
"Used" float NULL,
"EffectiveFree" float NULL,
"Object_UUID" uuid NOT NULL,
ts timestamptz NOT NULL
,CONSTRAINT PK_vsan_storage PRIMARY KEY (ts, "Object_UUID")
);
--ALTER TABLE vmreports.vsan_storage OWNER TO dbo;
GRANT ALL ON vmreports.vsan_storage TO nifi;
--select * from vmreports.vsan_storage;
/*
{
"type": "record",
"name": "vmreports.vsan_storage",
"namespace": "org.apache.nifi",
"fields": [
{"name":"Object_UUID", "type":{"type": "string", "logicalType": "uuid"}},
{"name":"ts","type":{"type": "long", "logicalType": "local-timestamp-millis"}},
{"name":"Name", "type":["string" ,"null"]},
{"name":"ClusterNew", "type":["string" ,"null"]},
{"name":"Total", "type":["float" ,"null"]},
{"name":"Usable", "type":["float" ,"null"]},
{"name":"Used", "type":["float" ,"null"]},
{"name":"EffectiveFree", "type":["float" ,"null"]}
]
}
*/
/*
{
"*": {
"objId":"[&1].Name",
"1":"[&1].ClusterNew",
"2":"[&1].Total",
"3":"[&1].Usable",
"4":"[&1].Used",
"5":"[&1].EffectiveFree",
"objUUID":"[&1].Object_UUID"
}
}
*/