Move packages to proejct/pkg to follow go project folder structure convention (#1190)

This commit is contained in:
Ta-Ching Chen
2019-05-31 16:28:55 +08:00
committed by GitHub
parent 1c5fd92ad6
commit a0e9a39511
196 changed files with 1672 additions and 1716 deletions
+5
View File
@@ -0,0 +1,5 @@
To compile protocol buffer definitions, you need to have installed the standard C++ implementation of protocol buffers and the Go compiler plugin, protoc-gen-go.
See [here](https://github.com/golang/protobuf) for installation instructions.
Run the following command within this directory:
`protoc --go_out=build/gen *.proto`
+305
View File
@@ -0,0 +1,305 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: spec.proto
package redisCache
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Request struct {
Method string `protobuf:"bytes,1,opt,name=Method,proto3" json:"Method,omitempty"`
URL map[string]string `protobuf:"bytes,2,rep,name=URL,proto3" json:"URL,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
Header map[string]string `protobuf:"bytes,3,rep,name=Header,proto3" json:"Header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
Host string `protobuf:"bytes,4,opt,name=Host,proto3" json:"Host,omitempty"`
Form map[string]string `protobuf:"bytes,5,rep,name=Form,proto3" json:"Form,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
PostForm map[string]string `protobuf:"bytes,6,rep,name=PostForm,proto3" json:"PostForm,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Request) Reset() { *m = Request{} }
func (m *Request) String() string { return proto.CompactTextString(m) }
func (*Request) ProtoMessage() {}
func (*Request) Descriptor() ([]byte, []int) {
return fileDescriptor_spec_ce9436f0047fa204, []int{0}
}
func (m *Request) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Request.Unmarshal(m, b)
}
func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Request.Marshal(b, m, deterministic)
}
func (dst *Request) XXX_Merge(src proto.Message) {
xxx_messageInfo_Request.Merge(dst, src)
}
func (m *Request) XXX_Size() int {
return xxx_messageInfo_Request.Size(m)
}
func (m *Request) XXX_DiscardUnknown() {
xxx_messageInfo_Request.DiscardUnknown(m)
}
var xxx_messageInfo_Request proto.InternalMessageInfo
func (m *Request) GetMethod() string {
if m != nil {
return m.Method
}
return ""
}
func (m *Request) GetURL() map[string]string {
if m != nil {
return m.URL
}
return nil
}
func (m *Request) GetHeader() map[string]string {
if m != nil {
return m.Header
}
return nil
}
func (m *Request) GetHost() string {
if m != nil {
return m.Host
}
return ""
}
func (m *Request) GetForm() map[string]string {
if m != nil {
return m.Form
}
return nil
}
func (m *Request) GetPostForm() map[string]string {
if m != nil {
return m.PostForm
}
return nil
}
type Response struct {
Status string `protobuf:"bytes,7,opt,name=Status,proto3" json:"Status,omitempty"`
StatusCode int32 `protobuf:"varint,8,opt,name=StatusCode,proto3" json:"StatusCode,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Response) Reset() { *m = Response{} }
func (m *Response) String() string { return proto.CompactTextString(m) }
func (*Response) ProtoMessage() {}
func (*Response) Descriptor() ([]byte, []int) {
return fileDescriptor_spec_ce9436f0047fa204, []int{1}
}
func (m *Response) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Response.Unmarshal(m, b)
}
func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Response.Marshal(b, m, deterministic)
}
func (dst *Response) XXX_Merge(src proto.Message) {
xxx_messageInfo_Response.Merge(dst, src)
}
func (m *Response) XXX_Size() int {
return xxx_messageInfo_Response.Size(m)
}
func (m *Response) XXX_DiscardUnknown() {
xxx_messageInfo_Response.DiscardUnknown(m)
}
var xxx_messageInfo_Response proto.InternalMessageInfo
func (m *Response) GetStatus() string {
if m != nil {
return m.Status
}
return ""
}
func (m *Response) GetStatusCode() int32 {
if m != nil {
return m.StatusCode
}
return 0
}
type UniqueRequest struct {
Req *Request `protobuf:"bytes,10,opt,name=Req,proto3" json:"Req,omitempty"`
Resp *Response `protobuf:"bytes,11,opt,name=Resp,proto3" json:"Resp,omitempty"`
Trigger string `protobuf:"bytes,12,opt,name=Trigger,proto3" json:"Trigger,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *UniqueRequest) Reset() { *m = UniqueRequest{} }
func (m *UniqueRequest) String() string { return proto.CompactTextString(m) }
func (*UniqueRequest) ProtoMessage() {}
func (*UniqueRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_spec_ce9436f0047fa204, []int{2}
}
func (m *UniqueRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_UniqueRequest.Unmarshal(m, b)
}
func (m *UniqueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_UniqueRequest.Marshal(b, m, deterministic)
}
func (dst *UniqueRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_UniqueRequest.Merge(dst, src)
}
func (m *UniqueRequest) XXX_Size() int {
return xxx_messageInfo_UniqueRequest.Size(m)
}
func (m *UniqueRequest) XXX_DiscardUnknown() {
xxx_messageInfo_UniqueRequest.DiscardUnknown(m)
}
var xxx_messageInfo_UniqueRequest proto.InternalMessageInfo
func (m *UniqueRequest) GetReq() *Request {
if m != nil {
return m.Req
}
return nil
}
func (m *UniqueRequest) GetResp() *Response {
if m != nil {
return m.Resp
}
return nil
}
func (m *UniqueRequest) GetTrigger() string {
if m != nil {
return m.Trigger
}
return ""
}
// We need this because we don't store the ReqUID in the above Message (UniqueRequest)
// and we don't store the UID there because the key in Redis is the ReqUID -- including it in the value would be
// unnecessary duplication
// but we need it here because we want to display to the user the ReqResponses for a given ReqUID
type RecordedEntry struct {
ReqUID string `protobuf:"bytes,13,opt,name=ReqUID,proto3" json:"ReqUID,omitempty"`
Req *Request `protobuf:"bytes,14,opt,name=Req,proto3" json:"Req,omitempty"`
Resp *Response `protobuf:"bytes,15,opt,name=Resp,proto3" json:"Resp,omitempty"`
Trigger string `protobuf:"bytes,16,opt,name=Trigger,proto3" json:"Trigger,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *RecordedEntry) Reset() { *m = RecordedEntry{} }
func (m *RecordedEntry) String() string { return proto.CompactTextString(m) }
func (*RecordedEntry) ProtoMessage() {}
func (*RecordedEntry) Descriptor() ([]byte, []int) {
return fileDescriptor_spec_ce9436f0047fa204, []int{3}
}
func (m *RecordedEntry) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_RecordedEntry.Unmarshal(m, b)
}
func (m *RecordedEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_RecordedEntry.Marshal(b, m, deterministic)
}
func (dst *RecordedEntry) XXX_Merge(src proto.Message) {
xxx_messageInfo_RecordedEntry.Merge(dst, src)
}
func (m *RecordedEntry) XXX_Size() int {
return xxx_messageInfo_RecordedEntry.Size(m)
}
func (m *RecordedEntry) XXX_DiscardUnknown() {
xxx_messageInfo_RecordedEntry.DiscardUnknown(m)
}
var xxx_messageInfo_RecordedEntry proto.InternalMessageInfo
func (m *RecordedEntry) GetReqUID() string {
if m != nil {
return m.ReqUID
}
return ""
}
func (m *RecordedEntry) GetReq() *Request {
if m != nil {
return m.Req
}
return nil
}
func (m *RecordedEntry) GetResp() *Response {
if m != nil {
return m.Resp
}
return nil
}
func (m *RecordedEntry) GetTrigger() string {
if m != nil {
return m.Trigger
}
return ""
}
func init() {
proto.RegisterType((*Request)(nil), "redisCache.Request")
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.FormEntry")
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.HeaderEntry")
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.PostFormEntry")
proto.RegisterMapType((map[string]string)(nil), "redisCache.Request.URLEntry")
proto.RegisterType((*Response)(nil), "redisCache.Response")
proto.RegisterType((*UniqueRequest)(nil), "redisCache.UniqueRequest")
proto.RegisterType((*RecordedEntry)(nil), "redisCache.RecordedEntry")
}
func init() { proto.RegisterFile("spec.proto", fileDescriptor_spec_ce9436f0047fa204) }
var fileDescriptor_spec_ce9436f0047fa204 = []byte{
// 392 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0x5f, 0xab, 0xd3, 0x30,
0x18, 0xc6, 0xe9, 0xda, 0x75, 0xdd, 0x5b, 0xab, 0x23, 0x8e, 0x11, 0x86, 0x7f, 0x6a, 0x41, 0xe8,
0x55, 0xc1, 0x09, 0xce, 0x3f, 0x78, 0xe3, 0x54, 0x26, 0x4c, 0x90, 0x68, 0x3f, 0x40, 0x5d, 0x5f,
0xb6, 0xa1, 0x2e, 0x6b, 0x92, 0x0a, 0x3b, 0x1f, 0xe3, 0x7c, 0xdb, 0x73, 0x77, 0x68, 0xd2, 0xee,
0x6c, 0xd0, 0x8b, 0xd3, 0xbb, 0xf7, 0x4d, 0x9e, 0xdf, 0x9b, 0xe7, 0x49, 0x5a, 0x00, 0x79, 0xc0,
0x75, 0x72, 0x10, 0x5c, 0x71, 0x02, 0x02, 0xf3, 0x9d, 0x5c, 0x64, 0xeb, 0x2d, 0x46, 0x37, 0x36,
0x0c, 0x18, 0x16, 0x25, 0x4a, 0x45, 0x26, 0xe0, 0x7e, 0x47, 0xb5, 0xe5, 0x39, 0xb5, 0x42, 0x2b,
0x1e, 0xb2, 0xba, 0x23, 0x09, 0xd8, 0x29, 0x5b, 0xd1, 0x5e, 0x68, 0xc7, 0xfe, 0xec, 0x49, 0x72,
0x47, 0x27, 0x35, 0x99, 0xa4, 0x6c, 0xf5, 0x65, 0xaf, 0xc4, 0x91, 0x55, 0x42, 0x32, 0x07, 0x77,
0x89, 0x59, 0x8e, 0x82, 0xda, 0x1a, 0x79, 0xde, 0x86, 0x18, 0x85, 0xa1, 0x6a, 0x39, 0x21, 0xe0,
0x2c, 0xb9, 0x54, 0xd4, 0xd1, 0xc7, 0xeb, 0x9a, 0xbc, 0x02, 0xe7, 0x2b, 0x17, 0xff, 0x68, 0x5f,
0x8f, 0x7a, 0xda, 0x36, 0xaa, 0xda, 0x37, 0x83, 0xb4, 0x94, 0x7c, 0x04, 0xef, 0x07, 0x97, 0x4a,
0x63, 0xae, 0xc6, 0x5e, 0xb4, 0x61, 0x8d, 0xc6, 0xa0, 0x27, 0x64, 0xfa, 0x06, 0xbc, 0x26, 0x0f,
0x19, 0x81, 0xfd, 0x07, 0x8f, 0xf5, 0x7d, 0x54, 0x25, 0x19, 0x43, 0xff, 0x7f, 0xf6, 0xb7, 0x44,
0xda, 0xd3, 0x6b, 0xa6, 0x79, 0xdf, 0x7b, 0x6b, 0x4d, 0xdf, 0x81, 0x7f, 0x16, 0xaa, 0x13, 0x3a,
0x87, 0xe1, 0xc9, 0x49, 0x27, 0xf0, 0x03, 0x04, 0x17, 0x31, 0xba, 0xc0, 0xd1, 0x27, 0xf0, 0x18,
0xca, 0x03, 0xdf, 0x4b, 0xac, 0xde, 0xfe, 0xa7, 0xca, 0x54, 0x29, 0xe9, 0xc0, 0xbc, 0xbd, 0xe9,
0xc8, 0x33, 0x00, 0x53, 0x2d, 0x78, 0x8e, 0xd4, 0x0b, 0xad, 0xb8, 0xcf, 0xce, 0x56, 0xa2, 0x2b,
0x08, 0xd2, 0xfd, 0xae, 0x28, 0xb1, 0xf9, 0x88, 0x5e, 0x82, 0xcd, 0xb0, 0xa0, 0x10, 0x5a, 0xb1,
0x3f, 0x7b, 0xdc, 0x72, 0xef, 0xac, 0xda, 0x27, 0x31, 0x38, 0xd5, 0xd9, 0xd4, 0xd7, 0xba, 0xf1,
0xa5, 0xce, 0x78, 0x62, 0x5a, 0x41, 0x28, 0x0c, 0x7e, 0x89, 0xdd, 0x66, 0x83, 0x82, 0x3e, 0xd0,
0xd6, 0x9a, 0x36, 0xba, 0xb6, 0x20, 0x60, 0xb8, 0xe6, 0x22, 0xc7, 0xdc, 0xa4, 0x9f, 0x80, 0xcb,
0xb0, 0x48, 0xbf, 0x7d, 0xa6, 0x81, 0x49, 0x61, 0xba, 0xc6, 0xd4, 0xc3, 0x7b, 0x9a, 0x7a, 0xd4,
0xc5, 0xd4, 0xe8, 0xc2, 0xd4, 0x6f, 0x57, 0xff, 0x63, 0xaf, 0x6f, 0x03, 0x00, 0x00, 0xff, 0xff,
0x0f, 0xb0, 0x49, 0xaf, 0x71, 0x03, 0x00, 0x00,
}
+126
View File
@@ -0,0 +1,126 @@
/*
Copyright 2018 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package redis
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"github.com/golang/protobuf/proto"
"github.com/gomodule/redigo/redis"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/pkg/redis/build/gen"
)
func NewClient() (redis.Conn, error) {
redisIP := os.Getenv("REDIS_SERVICE_HOST") // TODO: Do this here or somewhere earlier?
redisPort := os.Getenv("REDIS_SERVICE_PORT")
if len(redisIP) == 0 || len(redisPort) == 0 {
return nil, errors.New("redis host or port not supplied")
}
redisURL := fmt.Sprintf("%s:%s", redisIP, redisPort)
c, err := redis.Dial("tcp", redisURL)
if err != nil {
return nil, errors.Wrapf(err, "could not connect to Redis at url %q", redisURL)
}
return c, nil
}
func Record(logger *zap.Logger, triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) {
// Case where the function should not have been recorded
if len(reqUID) == 0 {
return
}
fullPath := originalUrl.String()
escPayload := string(json.RawMessage(payload))
client, err := NewClient()
if err != nil {
logger.Error("could not create redis client", zap.Error(err))
return
}
url := make(map[string]string)
url["Host"] = request.URL.Host
url["Path"] = fullPath
url["Payload"] = escPayload
header := make(map[string]string)
for key, value := range request.Header {
header[key] = strings.Join(value, ",")
}
form := make(map[string]string)
for key, value := range request.Form {
form[key] = strings.Join(value, ",")
}
postForm := make(map[string]string)
for key, value := range request.PostForm {
postForm[key] = strings.Join(value, ",")
}
req := &redisCache.Request{
Method: request.Method,
URL: url,
Header: header,
Host: request.Host, // Proxied host?
Form: form,
PostForm: postForm,
}
logger.Info("storing request", zap.Any("request", req))
resp := &redisCache.Response{
Status: response.Status,
StatusCode: int32(response.StatusCode),
}
ureq := &redisCache.UniqueRequest{
Req: req,
Resp: resp,
Trigger: triggerName,
}
data, err := proto.Marshal(ureq)
if err != nil {
logger.Error("error marshalling request", zap.Error(err))
return
}
_, err = client.Do("HMSET", reqUID, "ReqResponse", data, "Timestamp", timestamp, "Trigger", triggerName)
if err != nil {
logger.Error("error saving request", zap.Error(err))
return
}
_, err = client.Do("LPUSH", recorderName, reqUID)
if err != nil {
logger.Error("error saving recorder-request pair", zap.Error(err))
return
}
}
+423
View File
@@ -0,0 +1,423 @@
/*
Copyright 2018 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package redis
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
"go.uber.org/zap"
"github.com/golang/protobuf/proto"
"github.com/gomodule/redigo/redis"
"github.com/pkg/errors"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/redis/build/gen"
)
func RecordsListAll(logger *zap.Logger) ([]byte, error) {
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
iter := 0
var filtered []*redisCache.RecordedEntry
for {
// Each scan yields only a subset of all keys which is why we keep an iter. When iter == 0,
// Redis tells us there are no keys left to traverse.
arr, err := redis.Values(client.Do("SCAN", iter))
if err != nil {
return nil, err
}
// SCAN return value is an array of two values: the first value is the new cursor to use in the next call,
// the second value is an array of elements.
iter, _ = redis.Int(arr[0], nil)
keys, _ := redis.Strings(arr[1], nil)
for _, key := range keys {
if strings.HasPrefix(key, "REQ") {
val, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
if err != nil {
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val, key)
if err != nil {
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
}
}
if iter == 0 {
break
}
}
resp, err := json.Marshal(filtered)
if err != nil {
return nil, err
}
return resp, nil
}
// Input: `from` (hours ago, between 0 [today] and 5) and `to` (same units)
// Note: Fractional values don't seem to work -- document that for the user
func RecordsFilterByTime(logger *zap.Logger, from string, to string) ([]byte, error) {
rangeStart, rangeEnd, err := obtainInterval(from, to)
if err != nil {
return nil, err
}
logger.Debug("interval inferred", zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd))
if rangeStart >= rangeEnd {
e := "invalid chronology - start is greater than or equal to end"
logger.Error(e, zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd))
return nil, errors.New(e)
}
client, err := NewClient()
if client == nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
iter := 0
var filtered []*redisCache.RecordedEntry
for {
arr, err := redis.Values(client.Do("SCAN", iter))
if err != nil {
return nil, err
}
// SCAN return value is an array of two values: the first value is the new cursor to use in the next call,
// the second value is an array of elements.
iter, _ = redis.Int(arr[0], nil)
keys, _ := redis.Strings(arr[1], nil)
for _, key := range keys {
if strings.HasPrefix(key, "REQ") {
val, err := redis.Strings(client.Do("HMGET", key, "Timestamp"))
if err != nil {
logger.Error("error retrieving timestamp from redis", zap.Error(err))
return nil, err
}
tsO, err := strconv.Atoi(val[0])
if err != nil {
logger.Error("error converting timestamp to int", zap.Error(err))
return nil, err
}
ts := int64(tsO)
if ts >= rangeStart && ts <= rangeEnd {
val2, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
if err != nil {
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val2, key)
if err != nil {
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
}
}
}
if iter == 0 {
break
}
}
resp, err := json.Marshal(filtered)
if err != nil {
return nil, err
}
return resp, nil
}
func RecordsFilterByTrigger(logger *zap.Logger, queriedTriggerName string, recorders *fv1.RecorderList, triggers *fv1.HTTPTriggerList) ([]byte, error) {
matchingRecorders := make(map[string]bool)
// Implicit triggers:
// Sometimes triggers are not explicitly attached to recorders but we still want to be able to
// filter records by those triggers; we do so by identifying the functionReference the queriedTriggerName trigger has
// and finding recorder(s) for that function
var correspFunction string
for _, trigger := range triggers.Items {
if trigger.Metadata.Name == queriedTriggerName {
correspFunction = trigger.Spec.FunctionReference.Name
break
}
}
for _, recorder := range recorders.Items {
if len(recorder.Spec.Triggers) > 0 {
if includesTrigger(recorder.Spec.Triggers, queriedTriggerName) {
matchingRecorders[recorder.Spec.Name] = true
}
}
if recorder.Spec.Function == correspFunction {
matchingRecorders[recorder.Spec.Name] = true
}
}
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
var filtered []*redisCache.RecordedEntry
// TODO: Account for old/not-yet-deleted entries in the recorder lists
for key := range matchingRecorders {
val, err := redis.Strings(client.Do("LRANGE", key, "0", "-1")) // TODO: Prefix that distinguishes recorder lists
if err != nil {
// TODO: Handle deleted recorder? Or is this a non-issue because our list of recorders is up to date?
return nil, err
}
for _, reqUID := range val {
val, err := redis.Strings(client.Do("HMGET", reqUID, "Trigger")) // 1-to-1 reqUID - trigger?
if err != nil {
logger.Error("error retrieving trigger for a request from redis", zap.Error(err))
return nil, err
}
if val[0] == queriedTriggerName {
// TODO: Reconsider multiple commands
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
if err != nil {
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val, reqUID)
if err != nil {
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
}
}
}
resp, err := json.Marshal(filtered)
if err != nil {
return nil, err
}
return resp, nil
}
func RecordsFilterByFunction(logger *zap.Logger, queriedFunctionName string, recorders *fv1.RecorderList, triggers *fv1.HTTPTriggerList) ([]byte, error) {
// Implicit functions:
// Sometimes functions are not explicitly attached to recorders but we still want to be able to
// filter records by those functions; we do so by identifying all triggers recorders are associated with
// and checking functionReferences for those triggers.
triggerMap := make(map[string]fv1.HTTPTrigger)
for _, trigger := range triggers.Items {
triggerMap[trigger.Metadata.Name] = trigger
}
matchingRecorders := make(map[string]bool)
for _, recorder := range recorders.Items {
if len(recorder.Spec.Function) > 0 && recorder.Spec.Function == queriedFunctionName {
matchingRecorders[recorder.Spec.Name] = true
} else {
for _, trigger := range recorder.Spec.Triggers {
validTrigger, ok := triggerMap[trigger]
if ok {
if validTrigger.Spec.FunctionReference.Name == queriedFunctionName {
matchingRecorders[recorder.Spec.Name] = true
}
}
}
}
}
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
var filtered []*redisCache.RecordedEntry
for key := range matchingRecorders {
val, err := redis.Strings(client.Do("LRANGE", key, "0", "-1")) // TODO: Prefix that distinguishes recorder lists
if err != nil {
return nil, err
}
for _, reqUID := range val {
// TODO: Check if it still exists, else clean up this value from the cache
exists, err := redis.Int(client.Do("EXISTS", reqUID))
if err != nil {
continue
}
if exists > 0 {
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
if err != nil {
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val, reqUID)
if err != nil {
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
}
}
}
resp, err := json.Marshal(filtered)
if err != nil {
return nil, err
}
return resp, nil
}
// TODO: Discuss this approach of using two different protobuf message formats
func deserializeReqResponse(value []byte, reqUID string) (*redisCache.RecordedEntry, error) {
data := &redisCache.UniqueRequest{}
err := proto.Unmarshal(value, data)
if err != nil {
return nil, errors.Wrap(err, "error unmarshalling request")
}
transformed := &redisCache.RecordedEntry{
ReqUID: reqUID,
Req: data.Req,
Resp: data.Resp,
Trigger: data.Trigger,
}
return transformed, nil
}
func obtainInterval(from string, to string) (int64, int64, error) {
now := time.Now()
parsedFrom, err := time.ParseDuration(from)
if err != nil {
return -1, -1, err
}
parsedTo, err := time.ParseDuration(to)
if err != nil {
return -1, -1, err
}
then := now.Add(-1 * parsedFrom) // Start search interval
rangeStart := then.UnixNano()
until := now.Add(-1 * parsedTo) // End search interval
rangeEnd := until.UnixNano()
return rangeStart, rangeEnd, nil
}
func includesTrigger(triggers []string, query string) bool {
for _, trigger := range triggers {
if trigger == query {
return true
}
}
return false
}
func ReplayByReqUID(logger *zap.Logger, routerUrl string, queriedID string) ([]byte, error) {
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
exists, err := redis.Int(client.Do("EXISTS", queriedID))
if exists != 1 || err != nil {
logger.Error("could not find request to replay in redis", zap.Error(err))
return nil, err
}
val, err := redis.Bytes(client.Do("HGET", queriedID, "ReqResponse"))
if err != nil {
logger.Error("could not obtain ReqResponse for ID from redis", zap.Error(err), zap.String("id", queriedID))
return nil, err
}
entry, err := deserializeReqResponse(val, queriedID)
if err != nil {
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
replayed, err := ReplayRequest(routerUrl, entry.Req)
if err != nil {
logger.Error("error replaying request", zap.Error(err))
return nil, err
}
resp, err := json.Marshal(replayed)
if err != nil {
logger.Error("error marshalling replayed request response", zap.Error(err))
return nil, err
}
return resp, nil
}
func ReplayRequest(routerUrl string, request *redisCache.Request) ([]string, error) {
path := request.URL["Path"] // Includes slash prefix
payload := request.URL["Payload"]
targetUrl := fmt.Sprintf("%v%v", routerUrl, path)
var req *http.Request
var err error
client := http.DefaultClient
if request.Method == http.MethodGet {
req, err = http.NewRequest("GET", targetUrl, nil)
if err != nil {
return nil, err
}
} else {
req, err = http.NewRequest(request.Method, targetUrl, bytes.NewReader([]byte(payload)))
if err != nil {
return nil, err
}
}
req.Header.Set("X-Fission-Replayed", "true")
resp, err := client.Do(req)
if err != nil {
return nil, errors.New(fmt.Sprintf("failed to make request: %v", err))
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.New(fmt.Sprintf("failed to read response: %v", err))
}
bodyStr := string(body)
return []string{bodyStr}, nil
}
+51
View File
@@ -0,0 +1,51 @@
/*
Copyright 2018 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
syntax = 'proto3';
package redisCache;
message Request {
string Method = 1;
map<string, string> URL = 2;
map<string, string> Header = 3;
string Host = 4;
map<string, string> Form = 5;
map<string, string> PostForm = 6;
}
message Response {
string Status = 7;
int32 StatusCode = 8;
}
message UniqueRequest {
Request Req = 10;
Response Resp = 11;
string Trigger = 12;
}
// We need this because we don't store the ReqUID in the above Message (UniqueRequest)
// and we don't store the UID there because the key in Redis is the ReqUID -- including it in the value would be
// unnecessary duplication
// but we need it here because we want to display to the user the ReqResponses for a given ReqUID
message RecordedEntry {
string ReqUID = 13;
Request Req = 14;
Response Resp = 15;
string Trigger = 16;
}