Recorder CRD, Records API, Redis deployment (#818)
This commit is contained in:
Nafisa Shazia
committed by
Ta-Ching Chen
Ta-Ching Chen
parent
07ca5df8d0
commit
74a3a54543
@@ -0,0 +1,5 @@
|
||||
To compile protocol buffer definitions, you need to have installed the standard C++ implementation of protocol buffers and the Go compiler plugin, protoc-gen-go.
|
||||
See [here](https://github.com/golang/protobuf) for installation instructions.
|
||||
|
||||
Run the following command within this directory:
|
||||
`protoc --go_out=build/gen *.proto`
|
||||
@@ -0,0 +1,305 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// source: spec.proto
|
||||
|
||||
package redisCache
|
||||
|
||||
import proto "github.com/golang/protobuf/proto"
|
||||
import fmt "fmt"
|
||||
import math "math"
|
||||
|
||||
// Reference imports to suppress errors if they are not otherwise used.
|
||||
var _ = proto.Marshal
|
||||
var _ = fmt.Errorf
|
||||
var _ = math.Inf
|
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the proto package it is being compiled against.
|
||||
// A compilation error at this line likely means your copy of the
|
||||
// proto package needs to be updated.
|
||||
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
|
||||
|
||||
type Request struct {
|
||||
Method string `protobuf:"bytes,1,opt,name=Method,proto3" json:"Method,omitempty"`
|
||||
URL map[string]string `protobuf:"bytes,2,rep,name=URL,proto3" json:"URL,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||
Header map[string]string `protobuf:"bytes,3,rep,name=Header,proto3" json:"Header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||
Host string `protobuf:"bytes,4,opt,name=Host,proto3" json:"Host,omitempty"`
|
||||
Form map[string]string `protobuf:"bytes,5,rep,name=Form,proto3" json:"Form,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||
PostForm map[string]string `protobuf:"bytes,6,rep,name=PostForm,proto3" json:"PostForm,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *Request) Reset() { *m = Request{} }
|
||||
func (m *Request) String() string { return proto.CompactTextString(m) }
|
||||
func (*Request) ProtoMessage() {}
|
||||
func (*Request) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_spec_ce9436f0047fa204, []int{0}
|
||||
}
|
||||
func (m *Request) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_Request.Unmarshal(m, b)
|
||||
}
|
||||
func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
return xxx_messageInfo_Request.Marshal(b, m, deterministic)
|
||||
}
|
||||
func (dst *Request) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_Request.Merge(dst, src)
|
||||
}
|
||||
func (m *Request) XXX_Size() int {
|
||||
return xxx_messageInfo_Request.Size(m)
|
||||
}
|
||||
func (m *Request) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_Request.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_Request proto.InternalMessageInfo
|
||||
|
||||
func (m *Request) GetMethod() string {
|
||||
if m != nil {
|
||||
return m.Method
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *Request) GetURL() map[string]string {
|
||||
if m != nil {
|
||||
return m.URL
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Request) GetHeader() map[string]string {
|
||||
if m != nil {
|
||||
return m.Header
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Request) GetHost() string {
|
||||
if m != nil {
|
||||
return m.Host
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *Request) GetForm() map[string]string {
|
||||
if m != nil {
|
||||
return m.Form
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Request) GetPostForm() map[string]string {
|
||||
if m != nil {
|
||||
return m.PostForm
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Status string `protobuf:"bytes,7,opt,name=Status,proto3" json:"Status,omitempty"`
|
||||
StatusCode int32 `protobuf:"varint,8,opt,name=StatusCode,proto3" json:"StatusCode,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *Response) Reset() { *m = Response{} }
|
||||
func (m *Response) String() string { return proto.CompactTextString(m) }
|
||||
func (*Response) ProtoMessage() {}
|
||||
func (*Response) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_spec_ce9436f0047fa204, []int{1}
|
||||
}
|
||||
func (m *Response) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_Response.Unmarshal(m, b)
|
||||
}
|
||||
func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
return xxx_messageInfo_Response.Marshal(b, m, deterministic)
|
||||
}
|
||||
func (dst *Response) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_Response.Merge(dst, src)
|
||||
}
|
||||
func (m *Response) XXX_Size() int {
|
||||
return xxx_messageInfo_Response.Size(m)
|
||||
}
|
||||
func (m *Response) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_Response.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_Response proto.InternalMessageInfo
|
||||
|
||||
func (m *Response) GetStatus() string {
|
||||
if m != nil {
|
||||
return m.Status
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *Response) GetStatusCode() int32 {
|
||||
if m != nil {
|
||||
return m.StatusCode
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type UniqueRequest struct {
|
||||
Req *Request `protobuf:"bytes,10,opt,name=Req,proto3" json:"Req,omitempty"`
|
||||
Resp *Response `protobuf:"bytes,11,opt,name=Resp,proto3" json:"Resp,omitempty"`
|
||||
Trigger string `protobuf:"bytes,12,opt,name=Trigger,proto3" json:"Trigger,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *UniqueRequest) Reset() { *m = UniqueRequest{} }
|
||||
func (m *UniqueRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*UniqueRequest) ProtoMessage() {}
|
||||
func (*UniqueRequest) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_spec_ce9436f0047fa204, []int{2}
|
||||
}
|
||||
func (m *UniqueRequest) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_UniqueRequest.Unmarshal(m, b)
|
||||
}
|
||||
func (m *UniqueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
return xxx_messageInfo_UniqueRequest.Marshal(b, m, deterministic)
|
||||
}
|
||||
func (dst *UniqueRequest) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_UniqueRequest.Merge(dst, src)
|
||||
}
|
||||
func (m *UniqueRequest) XXX_Size() int {
|
||||
return xxx_messageInfo_UniqueRequest.Size(m)
|
||||
}
|
||||
func (m *UniqueRequest) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_UniqueRequest.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_UniqueRequest proto.InternalMessageInfo
|
||||
|
||||
func (m *UniqueRequest) GetReq() *Request {
|
||||
if m != nil {
|
||||
return m.Req
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *UniqueRequest) GetResp() *Response {
|
||||
if m != nil {
|
||||
return m.Resp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *UniqueRequest) GetTrigger() string {
|
||||
if m != nil {
|
||||
return m.Trigger
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// We need this because we don't store the ReqUID in the above Message (UniqueRequest)
|
||||
// and we don't store the UID there because the key in Redis is the ReqUID -- including it in the value would be
|
||||
// unnecessary duplication
|
||||
// but we need it here because we want to display to the user the ReqResponses for a given ReqUID
|
||||
type RecordedEntry struct {
|
||||
ReqUID string `protobuf:"bytes,13,opt,name=ReqUID,proto3" json:"ReqUID,omitempty"`
|
||||
Req *Request `protobuf:"bytes,14,opt,name=Req,proto3" json:"Req,omitempty"`
|
||||
Resp *Response `protobuf:"bytes,15,opt,name=Resp,proto3" json:"Resp,omitempty"`
|
||||
Trigger string `protobuf:"bytes,16,opt,name=Trigger,proto3" json:"Trigger,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *RecordedEntry) Reset() { *m = RecordedEntry{} }
|
||||
func (m *RecordedEntry) String() string { return proto.CompactTextString(m) }
|
||||
func (*RecordedEntry) ProtoMessage() {}
|
||||
func (*RecordedEntry) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_spec_ce9436f0047fa204, []int{3}
|
||||
}
|
||||
func (m *RecordedEntry) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_RecordedEntry.Unmarshal(m, b)
|
||||
}
|
||||
func (m *RecordedEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
return xxx_messageInfo_RecordedEntry.Marshal(b, m, deterministic)
|
||||
}
|
||||
func (dst *RecordedEntry) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_RecordedEntry.Merge(dst, src)
|
||||
}
|
||||
func (m *RecordedEntry) XXX_Size() int {
|
||||
return xxx_messageInfo_RecordedEntry.Size(m)
|
||||
}
|
||||
func (m *RecordedEntry) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_RecordedEntry.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_RecordedEntry proto.InternalMessageInfo
|
||||
|
||||
func (m *RecordedEntry) GetReqUID() string {
|
||||
if m != nil {
|
||||
return m.ReqUID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *RecordedEntry) GetReq() *Request {
|
||||
if m != nil {
|
||||
return m.Req
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RecordedEntry) GetResp() *Response {
|
||||
if m != nil {
|
||||
return m.Resp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RecordedEntry) GetTrigger() string {
|
||||
if m != nil {
|
||||
return m.Trigger
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func init() {
|
||||
proto.RegisterType((*Request)(nil), "redisCache.Request")
|
||||
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.FormEntry")
|
||||
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.HeaderEntry")
|
||||
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.PostFormEntry")
|
||||
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.URLEntry")
|
||||
proto.RegisterType((*Response)(nil), "redisCache.Response")
|
||||
proto.RegisterType((*UniqueRequest)(nil), "redisCache.UniqueRequest")
|
||||
proto.RegisterType((*RecordedEntry)(nil), "redisCache.RecordedEntry")
|
||||
}
|
||||
|
||||
func init() { proto.RegisterFile("spec.proto", fileDescriptor_spec_ce9436f0047fa204) }
|
||||
|
||||
var fileDescriptor_spec_ce9436f0047fa204 = []byte{
|
||||
// 392 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0x5f, 0xab, 0xd3, 0x30,
|
||||
0x18, 0xc6, 0xe9, 0xda, 0x75, 0xdd, 0x5b, 0xab, 0x23, 0x8e, 0x11, 0x86, 0x7f, 0x6a, 0x41, 0xe8,
|
||||
0x55, 0xc1, 0x09, 0xce, 0x3f, 0x78, 0xe3, 0x54, 0x26, 0x4c, 0x90, 0x68, 0x3f, 0x40, 0x5d, 0x5f,
|
||||
0xb6, 0xa1, 0x2e, 0x6b, 0x92, 0x0a, 0x3b, 0x1f, 0xe3, 0x7c, 0xdb, 0x73, 0x77, 0x68, 0xd2, 0xee,
|
||||
0x6c, 0xd0, 0x8b, 0xd3, 0xbb, 0xf7, 0x4d, 0x9e, 0xdf, 0x9b, 0xe7, 0x49, 0x5a, 0x00, 0x79, 0xc0,
|
||||
0x75, 0x72, 0x10, 0x5c, 0x71, 0x02, 0x02, 0xf3, 0x9d, 0x5c, 0x64, 0xeb, 0x2d, 0x46, 0x37, 0x36,
|
||||
0x0c, 0x18, 0x16, 0x25, 0x4a, 0x45, 0x26, 0xe0, 0x7e, 0x47, 0xb5, 0xe5, 0x39, 0xb5, 0x42, 0x2b,
|
||||
0x1e, 0xb2, 0xba, 0x23, 0x09, 0xd8, 0x29, 0x5b, 0xd1, 0x5e, 0x68, 0xc7, 0xfe, 0xec, 0x49, 0x72,
|
||||
0x47, 0x27, 0x35, 0x99, 0xa4, 0x6c, 0xf5, 0x65, 0xaf, 0xc4, 0x91, 0x55, 0x42, 0x32, 0x07, 0x77,
|
||||
0x89, 0x59, 0x8e, 0x82, 0xda, 0x1a, 0x79, 0xde, 0x86, 0x18, 0x85, 0xa1, 0x6a, 0x39, 0x21, 0xe0,
|
||||
0x2c, 0xb9, 0x54, 0xd4, 0xd1, 0xc7, 0xeb, 0x9a, 0xbc, 0x02, 0xe7, 0x2b, 0x17, 0xff, 0x68, 0x5f,
|
||||
0x8f, 0x7a, 0xda, 0x36, 0xaa, 0xda, 0x37, 0x83, 0xb4, 0x94, 0x7c, 0x04, 0xef, 0x07, 0x97, 0x4a,
|
||||
0x63, 0xae, 0xc6, 0x5e, 0xb4, 0x61, 0x8d, 0xc6, 0xa0, 0x27, 0x64, 0xfa, 0x06, 0xbc, 0x26, 0x0f,
|
||||
0x19, 0x81, 0xfd, 0x07, 0x8f, 0xf5, 0x7d, 0x54, 0x25, 0x19, 0x43, 0xff, 0x7f, 0xf6, 0xb7, 0x44,
|
||||
0xda, 0xd3, 0x6b, 0xa6, 0x79, 0xdf, 0x7b, 0x6b, 0x4d, 0xdf, 0x81, 0x7f, 0x16, 0xaa, 0x13, 0x3a,
|
||||
0x87, 0xe1, 0xc9, 0x49, 0x27, 0xf0, 0x03, 0x04, 0x17, 0x31, 0xba, 0xc0, 0xd1, 0x27, 0xf0, 0x18,
|
||||
0xca, 0x03, 0xdf, 0x4b, 0xac, 0xde, 0xfe, 0xa7, 0xca, 0x54, 0x29, 0xe9, 0xc0, 0xbc, 0xbd, 0xe9,
|
||||
0xc8, 0x33, 0x00, 0x53, 0x2d, 0x78, 0x8e, 0xd4, 0x0b, 0xad, 0xb8, 0xcf, 0xce, 0x56, 0xa2, 0x2b,
|
||||
0x08, 0xd2, 0xfd, 0xae, 0x28, 0xb1, 0xf9, 0x88, 0x5e, 0x82, 0xcd, 0xb0, 0xa0, 0x10, 0x5a, 0xb1,
|
||||
0x3f, 0x7b, 0xdc, 0x72, 0xef, 0xac, 0xda, 0x27, 0x31, 0x38, 0xd5, 0xd9, 0xd4, 0xd7, 0xba, 0xf1,
|
||||
0xa5, 0xce, 0x78, 0x62, 0x5a, 0x41, 0x28, 0x0c, 0x7e, 0x89, 0xdd, 0x66, 0x83, 0x82, 0x3e, 0xd0,
|
||||
0xd6, 0x9a, 0x36, 0xba, 0xb6, 0x20, 0x60, 0xb8, 0xe6, 0x22, 0xc7, 0xdc, 0xa4, 0x9f, 0x80, 0xcb,
|
||||
0xb0, 0x48, 0xbf, 0x7d, 0xa6, 0x81, 0x49, 0x61, 0xba, 0xc6, 0xd4, 0xc3, 0x7b, 0x9a, 0x7a, 0xd4,
|
||||
0xc5, 0xd4, 0xe8, 0xc2, 0xd4, 0x6f, 0x57, 0xff, 0x63, 0xaf, 0x6f, 0x03, 0x00, 0x00, 0xff, 0xff,
|
||||
0x0f, 0xb0, 0x49, 0xaf, 0x71, 0x03, 0x00, 0x00,
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
Copyright 2018 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package redis
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/gomodule/redigo/redis"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/fission/fission/redis/build/gen"
|
||||
)
|
||||
|
||||
func NewClient() redis.Conn {
|
||||
redisIP := os.Getenv("REDIS_SERVICE_HOST") // TODO: Do this here or somewhere earlier?
|
||||
redisPort := os.Getenv("REDIS_SERVICE_PORT")
|
||||
redisUrl := fmt.Sprintf("%s:%s", redisIP, redisPort)
|
||||
|
||||
if len(redisUrl) == 0 {
|
||||
log.Error("Could not reach Redis in cluster at IP ", redisUrl)
|
||||
return nil
|
||||
}
|
||||
|
||||
c, err := redis.Dial("tcp", redisUrl)
|
||||
if err != nil {
|
||||
log.Error("Could not connect to Redis: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func Record(triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) {
|
||||
// Case where the function should not have been recorded
|
||||
if len(reqUID) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
fullPath := originalUrl.String()
|
||||
escPayload := string(json.RawMessage(payload))
|
||||
|
||||
client := NewClient()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
|
||||
url := make(map[string]string)
|
||||
url["Host"] = request.URL.Host
|
||||
url["Path"] = fullPath
|
||||
url["Payload"] = escPayload
|
||||
|
||||
header := make(map[string]string)
|
||||
for key, value := range request.Header {
|
||||
header[key] = strings.Join(value, ",")
|
||||
}
|
||||
|
||||
form := make(map[string]string)
|
||||
for key, value := range request.Form {
|
||||
form[key] = strings.Join(value, ",")
|
||||
}
|
||||
|
||||
postForm := make(map[string]string)
|
||||
for key, value := range request.PostForm {
|
||||
postForm[key] = strings.Join(value, ",")
|
||||
}
|
||||
|
||||
req := &redisCache.Request{
|
||||
Method: request.Method,
|
||||
URL: url,
|
||||
Header: header,
|
||||
Host: request.Host, // Proxied host?
|
||||
Form: form,
|
||||
PostForm: postForm,
|
||||
}
|
||||
|
||||
resp := &redisCache.Response{
|
||||
Status: response.Status,
|
||||
StatusCode: int32(response.StatusCode),
|
||||
}
|
||||
|
||||
ureq := &redisCache.UniqueRequest{
|
||||
Req: req,
|
||||
Resp: resp,
|
||||
Trigger: triggerName,
|
||||
}
|
||||
|
||||
data, err := proto.Marshal(ureq)
|
||||
if err != nil {
|
||||
log.Error("Error marshalling request: ", err)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = client.Do("HMSET", reqUID, "ReqResponse", data, "Timestamp", timestamp, "Trigger", triggerName)
|
||||
if err != nil {
|
||||
log.Error("Error saving request: ", err)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = client.Do("LPUSH", recorderName, reqUID)
|
||||
if err != nil {
|
||||
log.Error("Error saving recorder-request pair: ", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
/*
|
||||
Copyright 2018 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package redis
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/gomodule/redigo/redis"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/fission/fission/crd"
|
||||
"github.com/fission/fission/redis/build/gen"
|
||||
)
|
||||
|
||||
func RecordsListAll() ([]byte, error) {
|
||||
client := NewClient()
|
||||
if client == nil {
|
||||
return []byte{}, errors.New("failed to create redis client")
|
||||
}
|
||||
|
||||
iter := 0
|
||||
var filtered []*redisCache.RecordedEntry
|
||||
|
||||
for {
|
||||
// Each scan yields only a subset of all keys which is why we keep an iter. When iter == 0,
|
||||
// Redis tells us there are no keys left to traverse.
|
||||
arr, err := redis.Values(client.Do("SCAN", iter))
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
// SCAN return value is an array of two values: the first value is the new cursor to use in the next call,
|
||||
// the second value is an array of elements.
|
||||
iter, _ = redis.Int(arr[0], nil)
|
||||
keys, _ := redis.Strings(arr[1], nil)
|
||||
for _, key := range keys {
|
||||
if strings.HasPrefix(key, "REQ") {
|
||||
val, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
|
||||
if err != nil {
|
||||
log.Error("Error retrieving request from Redis: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
entry, err := deserializeReqResponse(val, key)
|
||||
if err != nil {
|
||||
log.Error("Error deserializing request: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
}
|
||||
if iter == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := json.Marshal(filtered)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Input: `from` (hours ago, between 0 [today] and 5) and `to` (same units)
|
||||
// Note: Fractional values don't seem to work -- document that for the user
|
||||
func RecordsFilterByTime(from string, to string) ([]byte, error) {
|
||||
rangeStart, rangeEnd, err := obtainInterval(from, to)
|
||||
log.Debug("Interval inferred: ", rangeStart, rangeEnd)
|
||||
|
||||
if rangeStart >= rangeEnd {
|
||||
log.Error("Invalid chronology")
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
client := NewClient()
|
||||
if client == nil {
|
||||
return []byte{}, errors.New("failed to create redis client")
|
||||
}
|
||||
|
||||
iter := 0
|
||||
var filtered []*redisCache.RecordedEntry
|
||||
|
||||
for {
|
||||
arr, err := redis.Values(client.Do("SCAN", iter))
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
// SCAN return value is an array of two values: the first value is the new cursor to use in the next call,
|
||||
// the second value is an array of elements.
|
||||
iter, _ = redis.Int(arr[0], nil)
|
||||
keys, _ := redis.Strings(arr[1], nil)
|
||||
for _, key := range keys {
|
||||
if strings.HasPrefix(key, "REQ") {
|
||||
val, err := redis.Strings(client.Do("HMGET", key, "Timestamp"))
|
||||
if err != nil {
|
||||
log.Error("Error retrieving timestamp from Redis: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
tsO, err := strconv.Atoi(val[0])
|
||||
if err != nil {
|
||||
log.Error("Error converting timestamp to int: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
ts := int64(tsO)
|
||||
if ts >= rangeStart && ts <= rangeEnd {
|
||||
val2, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
|
||||
if err != nil {
|
||||
log.Error("Error retrieving request from Redis: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
entry, err := deserializeReqResponse(val2, key)
|
||||
if err != nil {
|
||||
log.Error("Error deserializing request: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if iter == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := json.Marshal(filtered)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
|
||||
matchingRecorders := make(map[string]bool)
|
||||
|
||||
// Implicit triggers:
|
||||
// Sometimes triggers are not explicitly attached to recorders but we still want to be able to
|
||||
// filter records by those triggers; we do so by identifying the functionReference the queriedTriggerName trigger has
|
||||
// and finding recorder(s) for that function
|
||||
|
||||
var correspFunction string
|
||||
for _, trigger := range triggers.Items {
|
||||
if trigger.Metadata.Name == queriedTriggerName {
|
||||
correspFunction = trigger.Spec.FunctionReference.Name
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, recorder := range recorders.Items {
|
||||
if len(recorder.Spec.Triggers) > 0 {
|
||||
if includesTrigger(recorder.Spec.Triggers, queriedTriggerName) {
|
||||
matchingRecorders[recorder.Spec.Name] = true
|
||||
}
|
||||
}
|
||||
if recorder.Spec.Function == correspFunction {
|
||||
matchingRecorders[recorder.Spec.Name] = true
|
||||
}
|
||||
}
|
||||
|
||||
client := NewClient()
|
||||
if client == nil {
|
||||
return []byte{}, errors.New("failed to create redis client")
|
||||
}
|
||||
|
||||
var filtered []*redisCache.RecordedEntry
|
||||
|
||||
// TODO: Account for old/not-yet-deleted entries in the recorder lists
|
||||
for key := range matchingRecorders {
|
||||
val, err := redis.Strings(client.Do("LRANGE", key, "0", "-1")) // TODO: Prefix that distinguishes recorder lists
|
||||
if err != nil {
|
||||
// TODO: Handle deleted recorder? Or is this a non-issue because our list of recorders is up to date?
|
||||
return []byte{}, err
|
||||
}
|
||||
for _, reqUID := range val {
|
||||
val, err := redis.Strings(client.Do("HMGET", reqUID, "Trigger")) // 1-to-1 reqUID - trigger?
|
||||
if err != nil {
|
||||
log.Error("Error retrieving trigger for a request from Redis: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
if val[0] == queriedTriggerName {
|
||||
// TODO: Reconsider multiple commands
|
||||
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
|
||||
if err != nil {
|
||||
log.Error("Error retrieving request from Redis: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
entry, err := deserializeReqResponse(val, reqUID)
|
||||
if err != nil {
|
||||
log.Error("Error deserializing request: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := json.Marshal(filtered)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
|
||||
|
||||
// Implicit functions:
|
||||
// Sometimes functions are not explicitly attached to recorders but we still want to be able to
|
||||
// filter records by those functions; we do so by identifying all triggers recorders are associated with
|
||||
// and checking functionReferences for those triggers.
|
||||
|
||||
triggerMap := make(map[string]crd.HTTPTrigger)
|
||||
for _, trigger := range triggers.Items {
|
||||
triggerMap[trigger.Metadata.Name] = trigger
|
||||
}
|
||||
|
||||
matchingRecorders := make(map[string]bool)
|
||||
|
||||
for _, recorder := range recorders.Items {
|
||||
if len(recorder.Spec.Function) > 0 && recorder.Spec.Function == queriedFunctionName {
|
||||
matchingRecorders[recorder.Spec.Name] = true
|
||||
} else {
|
||||
for _, trigger := range recorder.Spec.Triggers {
|
||||
validTrigger, ok := triggerMap[trigger]
|
||||
if ok {
|
||||
if validTrigger.Spec.FunctionReference.Name == queriedFunctionName {
|
||||
matchingRecorders[recorder.Spec.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
client := NewClient()
|
||||
if client == nil {
|
||||
return []byte{}, errors.New("failed to create redis client")
|
||||
}
|
||||
|
||||
var filtered []*redisCache.RecordedEntry
|
||||
|
||||
for key := range matchingRecorders {
|
||||
val, err := redis.Strings(client.Do("LRANGE", key, "0", "-1")) // TODO: Prefix that distinguishes recorder lists
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
for _, reqUID := range val {
|
||||
// TODO: Check if it still exists, else clean up this value from the cache
|
||||
exists, err := redis.Int(client.Do("EXISTS", reqUID))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if exists > 0 {
|
||||
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
|
||||
if err != nil {
|
||||
log.Error("Error retrieving request from Redis: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
entry, err := deserializeReqResponse(val, reqUID)
|
||||
if err != nil {
|
||||
log.Error("Error deserializing request: ", err)
|
||||
return []byte{}, err
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := json.Marshal(filtered)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// TODO: Discuss this approach of using two different protobuf message formats
|
||||
func deserializeReqResponse(value []byte, reqUID string) (*redisCache.RecordedEntry, error) {
|
||||
data := &redisCache.UniqueRequest{}
|
||||
err := proto.Unmarshal(value, data)
|
||||
if err != nil {
|
||||
log.Error("Error unmarshalling request: ", err)
|
||||
return nil, err
|
||||
}
|
||||
log.Info("Parsed protobuf bytes: ", data)
|
||||
transformed := &redisCache.RecordedEntry{
|
||||
ReqUID: reqUID,
|
||||
Req: data.Req,
|
||||
Resp: data.Resp,
|
||||
Trigger: data.Trigger,
|
||||
}
|
||||
return transformed, nil
|
||||
}
|
||||
|
||||
func obtainInterval(from string, to string) (int64, int64, error) {
|
||||
now := time.Now()
|
||||
parsedFrom, err := time.ParseDuration(from)
|
||||
if err != nil {
|
||||
return -1, -1, err
|
||||
}
|
||||
|
||||
parsedTo, err := time.ParseDuration(to)
|
||||
if err != nil {
|
||||
return -1, -1, err
|
||||
}
|
||||
|
||||
then := now.Add(-1 * parsedFrom) // Start search interval
|
||||
rangeStart := then.UnixNano()
|
||||
|
||||
until := now.Add(-1 * parsedTo) // End search interval
|
||||
rangeEnd := until.UnixNano()
|
||||
|
||||
return rangeStart, rangeEnd, nil
|
||||
}
|
||||
|
||||
func includesTrigger(triggers []string, query string) bool {
|
||||
for _, trigger := range triggers {
|
||||
if trigger == query {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
Copyright 2018 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
syntax = 'proto3';
|
||||
|
||||
package redisCache;
|
||||
|
||||
message Request {
|
||||
string Method = 1;
|
||||
map<string, string> URL = 2;
|
||||
map<string, string> Header = 3;
|
||||
string Host = 4;
|
||||
map<string, string> Form = 5;
|
||||
map<string, string> PostForm = 6;
|
||||
}
|
||||
|
||||
message Response {
|
||||
string Status = 7;
|
||||
int32 StatusCode = 8;
|
||||
}
|
||||
|
||||
message UniqueRequest {
|
||||
Request Req = 10;
|
||||
Response Resp = 11;
|
||||
string Trigger = 12;
|
||||
}
|
||||
|
||||
// We need this because we don't store the ReqUID in the above Message (UniqueRequest)
|
||||
// and we don't store the UID there because the key in Redis is the ReqUID -- including it in the value would be
|
||||
// unnecessary duplication
|
||||
// but we need it here because we want to display to the user the ReqResponses for a given ReqUID
|
||||
message RecordedEntry {
|
||||
string ReqUID = 13;
|
||||
Request Req = 14;
|
||||
Response Resp = 15;
|
||||
string Trigger = 16;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user