admin-console: implement Dynamic client CRD counting, S3 storage metrics, CSV export
This commit is contained in:
“Naeel”
@@ -1,7 +1,7 @@
|
||||
FROM golang:1.26-alpine AS builder
|
||||
WORKDIR /build
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY go.mod ./
|
||||
RUN go mod tidy
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o admin-console ./cmd/server/
|
||||
|
||||
|
||||
@@ -67,6 +67,20 @@ spec:
|
||||
secretKeyRef:
|
||||
name: fission-admin-token
|
||||
key: token
|
||||
- name: S3_ENDPOINT
|
||||
value: "https://s3.msk-1.ngcloud.ru"
|
||||
- name: S3_BUCKET
|
||||
value: "sless-functions"
|
||||
- name: S3_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: fission-s3-credentials
|
||||
key: access_key
|
||||
- name: S3_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: fission-s3-credentials
|
||||
key: secret_key
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
+7
-24
@@ -3,30 +3,13 @@ module admin-console
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
k8s.io/client-go v0.31.0
|
||||
k8s.io/apimachinery v0.31.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.0
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.0
|
||||
k8s.io/api v0.31.0
|
||||
k8s.io/apimachinery v0.31.0
|
||||
k8s.io/client-go v0.31.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/gofuzz v1.2.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
golang.org/x/net v0.26.0 // indirect
|
||||
golang.org/x/oauth2 v0.21.0 // indirect
|
||||
golang.org/x/term v0.21.0 // indirect
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
k8s.io/klog/v2 v2.130.1 // indirect
|
||||
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
|
||||
sigs.k8s.io/json v0.0.0-20221116144056-70938f8a5b36 // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
|
||||
sigs.k8s.io/yaml v1.4.0 // indirect
|
||||
)
|
||||
|
||||
|
||||
@@ -3,10 +3,20 @@ package metrics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
@@ -14,9 +24,21 @@ import (
|
||||
"admin-console/internal/model"
|
||||
)
|
||||
|
||||
// fission CRD group-version-resources.
|
||||
var (
|
||||
gvrFunctions = schema.GroupVersionResource{Group: "fission.io", Version: "v1", Resource: "functions"}
|
||||
gvrPackages = schema.GroupVersionResource{Group: "fission.io", Version: "v1", Resource: "packages"}
|
||||
gvrEnvironments = schema.GroupVersionResource{Group: "fission.io", Version: "v1", Resource: "environments"}
|
||||
gvrHTTPTriggers = schema.GroupVersionResource{Group: "fission.io", Version: "v1", Resource: "httptriggers"}
|
||||
gvrTimeTriggers = schema.GroupVersionResource{Group: "fission.io", Version: "v1", Resource: "timetriggers"}
|
||||
)
|
||||
|
||||
// K8sClient — обёртка над kubernetes.Clientset для нужд admin-console.
|
||||
type K8sClient struct {
|
||||
cs *kubernetes.Clientset
|
||||
cs *kubernetes.Clientset
|
||||
dc dynamic.Interface
|
||||
s3 *s3.Client
|
||||
s3Bucket string
|
||||
}
|
||||
|
||||
// NewK8sClient создаёт клиента из in-cluster конфига, с fallback на ~/.kube/config.
|
||||
@@ -32,12 +54,50 @@ func NewK8sClient() (*K8sClient, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &K8sClient{cs: cs}, nil
|
||||
dc, err := dynamic.NewForConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dynamic client: %w", err)
|
||||
}
|
||||
|
||||
k := &K8sClient{cs: cs, dc: dc}
|
||||
|
||||
// S3 — опционально (не падаем если нет кредов)
|
||||
k.initS3()
|
||||
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// initS3 создаёт S3-клиент из переменных окружения.
|
||||
func (k *K8sClient) initS3() {
|
||||
endpoint := os.Getenv("S3_ENDPOINT")
|
||||
bucket := os.Getenv("S3_BUCKET")
|
||||
accessKey := os.Getenv("S3_ACCESS_KEY")
|
||||
secretKey := os.Getenv("S3_SECRET_KEY")
|
||||
|
||||
if endpoint == "" || bucket == "" || accessKey == "" || secretKey == "" {
|
||||
log.Println("[admin-console] S3 env vars not set — storage metrics disabled")
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion("ru-msk-1"),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("[admin-console] S3 config error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
k.s3 = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
k.s3Bucket = bucket
|
||||
log.Printf("[admin-console] S3 client ready, bucket=%s", bucket)
|
||||
}
|
||||
|
||||
// ListClientNamespaces возвращает namespace-ы с лейблом managed-by=fission-console.
|
||||
// Для каждого считает количество объектов Fission CRD через Dynamic клиент
|
||||
// (пока упрощённо — только через secrets и labels).
|
||||
// Для каждого считает количество объектов Fission CRD через Dynamic клиент.
|
||||
func (k *K8sClient) ListClientNamespaces(ctx context.Context) ([]model.NamespaceInfo, error) {
|
||||
nsList, err := k.cs.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
|
||||
LabelSelector: "managed-by=fission-console",
|
||||
@@ -53,12 +113,37 @@ func (k *K8sClient) ListClientNamespaces(ctx context.Context) ([]model.Namespace
|
||||
UserEmail: ns.Labels["user"],
|
||||
CreatedAt: ns.CreationTimestamp.Format(time.RFC3339),
|
||||
}
|
||||
// TODO: добавить подсчёт Fission CRD через dynamic client
|
||||
k.fillCRDCounts(ctx, ns.Name, &info)
|
||||
result = append(result, info)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// fillCRDCounts заполняет счётчики Fission-ресурсов для одного namespace.
|
||||
func (k *K8sClient) fillCRDCounts(ctx context.Context, ns string, info *model.NamespaceInfo) {
|
||||
count := func(gvr schema.GroupVersionResource) int {
|
||||
list, err := k.dc.Resource(gvr).Namespace(ns).List(ctx, metav1.ListOptions{Limit: 1})
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
// List возвращает items и list-metadata с remaining count.
|
||||
// Используем оставшийся счёт для точности.
|
||||
if rem := list.GetRemainingItemCount(); rem != nil {
|
||||
return int(*rem) + len(list.Items)
|
||||
}
|
||||
return len(list.Items)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(5)
|
||||
go func() { defer wg.Done(); info.Functions = count(gvrFunctions) }()
|
||||
go func() { defer wg.Done(); info.Packages = count(gvrPackages) }()
|
||||
go func() { defer wg.Done(); info.Environments = count(gvrEnvironments) }()
|
||||
go func() { defer wg.Done(); info.HTTPTriggers = count(gvrHTTPTriggers) }()
|
||||
go func() { defer wg.Done(); info.TimeTriggers = count(gvrTimeTriggers) }()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// CollectUsage возвращает агрегированную статистику по всем клиентским namespace-ам.
|
||||
func (k *K8sClient) CollectUsage(ctx context.Context) ([]model.UsageEntry, error) {
|
||||
nsList, err := k.cs.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
|
||||
@@ -79,12 +164,52 @@ func (k *K8sClient) CollectUsage(ctx context.Context) ([]model.UsageEntry, error
|
||||
PeriodStart: startOfMonth.Format("2006-01-02"),
|
||||
PeriodEnd: now.Format("2006-01-02"),
|
||||
}
|
||||
// TODO: заполнить реальными данными из Prometheus / metrics-server / S3
|
||||
|
||||
// Считаем функции через Dynamic client.
|
||||
if list, err := k.dc.Resource(gvrFunctions).Namespace(ns.Name).List(ctx, metav1.ListOptions{Limit: 1}); err == nil {
|
||||
entry.Functions = len(list.Items)
|
||||
if rem := list.GetRemainingItemCount(); rem != nil {
|
||||
entry.Functions += int(*rem)
|
||||
}
|
||||
}
|
||||
|
||||
// Считаем S3-хранилище.
|
||||
entry.StorageGB = k.namespaceStorage(ctx, ns.Name)
|
||||
|
||||
// Invocations / CPU / Memory — TODO.
|
||||
result = append(result, entry)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// namespaceStorage возвращает суммарный размер объектов S3 с префиксом "{ns}/".
|
||||
func (k *K8sClient) namespaceStorage(ctx context.Context, ns string) float64 {
|
||||
if k.s3 == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
var totalBytes int64
|
||||
paginator := s3.NewListObjectsV2Paginator(k.s3, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(k.s3Bucket),
|
||||
Prefix: aws.String(ns + "/"),
|
||||
})
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[admin-console] S3 list error ns=%s: %v", ns, err)
|
||||
return 0
|
||||
}
|
||||
for _, obj := range page.Contents {
|
||||
if obj.Size != nil {
|
||||
totalBytes += *obj.Size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return float64(totalBytes) / (1024 * 1024 * 1024)
|
||||
}
|
||||
|
||||
// ListUsers собирает пользователей из secret fission-console-users в каждом
|
||||
// клиентском namespace.
|
||||
func (k *K8sClient) ListUsers(ctx context.Context) ([]model.UserInfo, error) {
|
||||
|
||||
@@ -12,8 +12,8 @@ type NamespaceInfo struct {
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
|
||||
// счётчики ресурсов Fission
|
||||
Functions int `json:"functions"`
|
||||
Packages int `json:"packages"`
|
||||
Functions int `json:"functions"`
|
||||
Packages int `json:"packages"`
|
||||
Environments int `json:"environments"`
|
||||
HTTPTriggers int `json:"http_triggers"`
|
||||
TimeTriggers int `json:"time_triggers"`
|
||||
@@ -21,15 +21,15 @@ type NamespaceInfo struct {
|
||||
|
||||
// UsageEntry — строка использования для одного namespace (для бухгалтерии).
|
||||
type UsageEntry struct {
|
||||
Namespace string `json:"namespace"`
|
||||
UserEmail string `json:"user_email,omitempty"`
|
||||
Functions int `json:"functions"`
|
||||
Invocations int64 `json:"invocations_total"` // TODO: из Prometheus/logs
|
||||
StorageGB float64 `json:"storage_gb"` // TODO: из storagesvc/S3
|
||||
CPUMilliCores int64 `json:"cpu_millicores"` // TODO: из metrics-server
|
||||
MemoryMB int64 `json:"memory_mb"` // TODO: из metrics-server
|
||||
PeriodStart string `json:"period_start"`
|
||||
PeriodEnd string `json:"period_end"`
|
||||
Namespace string `json:"namespace"`
|
||||
UserEmail string `json:"user_email,omitempty"`
|
||||
Functions int `json:"functions"`
|
||||
Invocations int64 `json:"invocations_total"` // TODO: из Prometheus/logs
|
||||
StorageGB float64 `json:"storage_gb"` // TODO: из storagesvc/S3
|
||||
CPUMilliCores int64 `json:"cpu_millicores"` // TODO: из metrics-server
|
||||
MemoryMB int64 `json:"memory_mb"` // TODO: из metrics-server
|
||||
PeriodStart string `json:"period_start"`
|
||||
PeriodEnd string `json:"period_end"`
|
||||
}
|
||||
|
||||
// UserInfo — пользователь из secret fission-console-users.
|
||||
|
||||
@@ -125,19 +125,38 @@
|
||||
|
||||
function renderUsage(rows) {
|
||||
if (!rows?.length) return '<p class="loader">Нет данных</p>';
|
||||
return `<table>
|
||||
const btn = `<div style="margin-bottom:12px">
|
||||
<button onclick="exportCSV()">📥 Экспорт CSV</button>
|
||||
</div>`;
|
||||
return btn + `<table>
|
||||
<tr><th>Namespace</th><th>Email</th><th>Функции</th><th>Вызовы</th><th>Хранилище GB</th><th>Период</th></tr>
|
||||
${rows.map(u => `<tr>
|
||||
<td><span class="pill pill-blue">${u.namespace}</span></td>
|
||||
<td>${u.user_email || '—'}</td>
|
||||
<td>${u.functions}</td>
|
||||
<td>${u.invocations_total}</td>
|
||||
<td>${u.storage_gb.toFixed(2)}</td>
|
||||
<td>${(u.storage_gb || 0).toFixed(3)}</td>
|
||||
<td>${u.period_start} — ${u.period_end}</td>
|
||||
</tr>`).join('')}
|
||||
</table>`;
|
||||
}
|
||||
|
||||
function exportCSV() {
|
||||
const table = document.querySelector('table');
|
||||
if (!table) return;
|
||||
let csv = '\uFEFF'; // BOM для Excel
|
||||
table.querySelectorAll('tr').forEach(row => {
|
||||
const cells = [];
|
||||
row.querySelectorAll('th,td').forEach(c => cells.push('"' + c.textContent.replace(/"/g, '""') + '"'));
|
||||
csv += cells.join(',') + '\n';
|
||||
});
|
||||
const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' });
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(blob);
|
||||
a.download = `usage-${new Date().toISOString().slice(0,10)}.csv`;
|
||||
a.click();
|
||||
}
|
||||
|
||||
function renderUsers(rows) {
|
||||
if (!rows?.length) return '<p class="loader">Нет пользователей</p>';
|
||||
return `<table>
|
||||
|
||||
Reference in New Issue
Block a user