537 lines
18 KiB
Go
537 lines
18 KiB
Go
/*
|
|
Copyright 2016 The Fission Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package router
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/bep/debounce"
|
|
"github.com/gorilla/mux"
|
|
"go.uber.org/zap"
|
|
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/kubernetes"
|
|
k8sCache "k8s.io/client-go/tools/cache"
|
|
|
|
fv1 "github.com/fission/fission/pkg/apis/core/v1"
|
|
ferror "github.com/fission/fission/pkg/error"
|
|
eclient "github.com/fission/fission/pkg/executor/client"
|
|
config "github.com/fission/fission/pkg/featureconfig"
|
|
"github.com/fission/fission/pkg/generated/clientset/versioned"
|
|
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
|
|
"github.com/fission/fission/pkg/info"
|
|
"github.com/fission/fission/pkg/throttler"
|
|
"github.com/fission/fission/pkg/utils"
|
|
"github.com/fission/fission/pkg/utils/manager"
|
|
"github.com/fission/fission/pkg/utils/metrics"
|
|
)
|
|
|
|
// HTTPTriggerSet represents an HTTP trigger set
|
|
type HTTPTriggerSet struct {
|
|
*functionServiceMap
|
|
*mutableRouter
|
|
|
|
logger *zap.Logger
|
|
fissionClient versioned.Interface
|
|
kubeClient kubernetes.Interface
|
|
executor eclient.ClientInterface
|
|
resolver *functionReferenceResolver
|
|
triggers []fv1.HTTPTrigger
|
|
triggerInformer map[string]k8sCache.SharedIndexInformer
|
|
functions []fv1.Function
|
|
funcInformer map[string]k8sCache.SharedIndexInformer
|
|
informerMu sync.RWMutex
|
|
updateRouterRequestChannel chan struct{}
|
|
tsRoundTripperParams *tsRoundTripperParams
|
|
isDebugEnv bool
|
|
svcAddrUpdateThrottler *throttler.Throttler
|
|
unTapServiceTimeout time.Duration
|
|
syncDebouncer func(func())
|
|
}
|
|
|
|
func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, fissionClient versioned.Interface,
|
|
kubeClient kubernetes.Interface, executor eclient.ClientInterface, params *tsRoundTripperParams, isDebugEnv bool, unTapServiceTimeout time.Duration, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, error) {
|
|
|
|
httpTriggerSet := &HTTPTriggerSet{
|
|
logger: logger.Named("http_trigger_set"),
|
|
functionServiceMap: fmap,
|
|
triggers: []fv1.HTTPTrigger{},
|
|
fissionClient: fissionClient,
|
|
kubeClient: kubeClient,
|
|
executor: executor,
|
|
updateRouterRequestChannel: make(chan struct{}, 10), // use buffer channel
|
|
tsRoundTripperParams: params,
|
|
isDebugEnv: isDebugEnv,
|
|
svcAddrUpdateThrottler: actionThrottler,
|
|
unTapServiceTimeout: unTapServiceTimeout,
|
|
syncDebouncer: debounce.New(time.Millisecond * 20),
|
|
}
|
|
httpTriggerSet.triggerInformer = utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.HttpTriggerResource)
|
|
httpTriggerSet.funcInformer = utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.FunctionResource)
|
|
err := httpTriggerSet.addTriggerHandlers()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
err = httpTriggerSet.addFunctionHandlers()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return httpTriggerSet, nil
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mgr manager.Interface, mr *mutableRouter) error {
|
|
resolver := makeFunctionReferenceResolver(ts.logger, ts.funcInformer)
|
|
ts.resolver = resolver
|
|
ts.mutableRouter = mr
|
|
|
|
if ts.fissionClient == nil {
|
|
// Used in tests only.
|
|
router, err := ts.getRouter(nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
mr.updateRouter(router)
|
|
ts.logger.Info("skipping continuous trigger updates")
|
|
return nil
|
|
}
|
|
mgr.Add(ctx, func(ctx context.Context) {
|
|
ts.updateRouter(ctx)
|
|
})
|
|
ts.syncTriggers()
|
|
mgr.AddInformers(ctx, ts.funcInformer)
|
|
mgr.AddInformers(ctx, ts.triggerInformer)
|
|
return nil
|
|
}
|
|
|
|
func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|
|
|
|
func routerHealthHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
}
|
|
|
|
func versionHandler(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
_, err := w.Write([]byte(info.ApiInfo().String()))
|
|
if err != nil {
|
|
se, ok := err.(*kerrors.StatusError)
|
|
if ok {
|
|
http.Error(w, se.Error(), int(se.ErrStatus.Code))
|
|
return
|
|
}
|
|
|
|
code, msg := ferror.GetHTTPError(err)
|
|
http.Error(w, msg, code)
|
|
}
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) getRouter(fnTimeoutMap map[types.UID]int) (*mux.Router, error) {
|
|
|
|
featureConfig, err := config.GetFeatureConfig(ts.logger)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
muxRouter := mux.NewRouter()
|
|
muxRouter.Use(metrics.HTTPMetricMiddleware)
|
|
if featureConfig.AuthConfig.IsEnabled {
|
|
muxRouter.Use(authMiddleware(featureConfig))
|
|
}
|
|
|
|
// HTTP triggers setup by the user
|
|
homeHandled := false
|
|
for i := range ts.triggers {
|
|
trigger := ts.triggers[i]
|
|
|
|
// resolve function reference
|
|
rr, err := ts.resolver.resolve(trigger)
|
|
if err != nil {
|
|
// Unresolvable function reference. Report the error via
|
|
// the trigger's status.
|
|
go ts.updateTriggerStatusFailed(&trigger, err)
|
|
|
|
// Ignore this route and let it 404.
|
|
continue
|
|
}
|
|
|
|
if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions {
|
|
// not implemented yet
|
|
ts.logger.Panic("resolve result type not implemented", zap.Any("type", rr.resolveResultType))
|
|
}
|
|
|
|
fh := &functionHandler{
|
|
logger: ts.logger.Named(trigger.ObjectMeta.Name),
|
|
fmap: ts.functionServiceMap,
|
|
executor: ts.executor,
|
|
httpTrigger: &trigger,
|
|
functionMap: rr.functionMap,
|
|
fnWeightDistributionList: rr.functionWtDistributionList,
|
|
tsRoundTripperParams: ts.tsRoundTripperParams,
|
|
isDebugEnv: ts.isDebugEnv,
|
|
svcAddrUpdateThrottler: ts.svcAddrUpdateThrottler,
|
|
functionTimeoutMap: fnTimeoutMap,
|
|
unTapServiceTimeout: ts.unTapServiceTimeout,
|
|
}
|
|
|
|
// The functionHandler for HTTP trigger with fn reference type "FunctionReferenceTypeFunctionName",
|
|
// it's function metadata is set here.
|
|
|
|
// The functionHandler For HTTP trigger with fn reference type "FunctionReferenceTypeFunctionWeights",
|
|
// it's function metadata is decided dynamically before proxying the request in order to support canary
|
|
// deployment. For more details, please check "handler" function of functionHandler.
|
|
|
|
if rr.resolveResultType == resolveResultSingleFunction {
|
|
for _, fn := range fh.functionMap {
|
|
fh.function = fn
|
|
}
|
|
}
|
|
|
|
methods := trigger.Spec.Methods
|
|
if len(trigger.Spec.Method) > 0 {
|
|
present := false
|
|
for _, m := range trigger.Spec.Methods {
|
|
if m == trigger.Spec.Method {
|
|
present = true
|
|
break
|
|
}
|
|
}
|
|
if !present {
|
|
methods = append(methods, trigger.Spec.Method)
|
|
}
|
|
}
|
|
|
|
handler := http.HandlerFunc(fh.handler)
|
|
|
|
if trigger.Spec.Prefix != nil && *trigger.Spec.Prefix != "" {
|
|
prefix := *trigger.Spec.Prefix
|
|
if strings.HasSuffix(prefix, "/") {
|
|
ht := muxRouter.PathPrefix(prefix).Handler(handler)
|
|
ht.Methods(methods...)
|
|
if trigger.Spec.Host != "" {
|
|
ht.Host(trigger.Spec.Host)
|
|
}
|
|
ts.logger.Debug("add prefix route for function", zap.String("route", prefix), zap.Any("function", fh.function), zap.Strings("methods", methods))
|
|
} else {
|
|
ht1 := muxRouter.Handle(prefix, handler)
|
|
ht1.Methods(methods...)
|
|
if trigger.Spec.Host != "" {
|
|
ht1.Host(trigger.Spec.Host)
|
|
}
|
|
ht2 := muxRouter.PathPrefix(prefix + "/").Handler(handler)
|
|
ht2.Methods(methods...)
|
|
if trigger.Spec.Host != "" {
|
|
ht2.Host(trigger.Spec.Host)
|
|
}
|
|
ts.logger.Debug("add prefix and handler route for function", zap.String("route", prefix), zap.Any("function", fh.function), zap.Strings("methods", methods))
|
|
}
|
|
} else {
|
|
ht := muxRouter.Handle(trigger.Spec.RelativeURL, handler)
|
|
ht.Methods(methods...)
|
|
if trigger.Spec.Host != "" {
|
|
ht.Host(trigger.Spec.Host)
|
|
}
|
|
ts.logger.Debug("add handler route for function", zap.String("router", trigger.Spec.RelativeURL), zap.Any("function", fh.function), zap.Strings("methods", methods))
|
|
}
|
|
|
|
if trigger.Spec.Prefix == nil && trigger.Spec.RelativeURL == "/" && len(methods) == 1 && methods[0] == http.MethodGet {
|
|
homeHandled = true
|
|
}
|
|
}
|
|
if !homeHandled {
|
|
//
|
|
// This adds a no-op handler that returns 200-OK to make sure that the
|
|
// "GET /" request succeeds. This route is used by GKE Ingress (and
|
|
// perhaps other ingress implementations) as a health check, so we don't
|
|
// want it to be a 404 even if the user doesn't have a function mapped to
|
|
// this route.
|
|
//
|
|
muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET")
|
|
}
|
|
|
|
// Internal triggers for each function by name. Non-http
|
|
// triggers route into these.
|
|
for i := range ts.functions {
|
|
fn := ts.functions[i]
|
|
fh := &functionHandler{
|
|
logger: ts.logger.Named(fn.ObjectMeta.Name),
|
|
fmap: ts.functionServiceMap,
|
|
function: &fn,
|
|
executor: ts.executor,
|
|
tsRoundTripperParams: ts.tsRoundTripperParams,
|
|
isDebugEnv: ts.isDebugEnv,
|
|
svcAddrUpdateThrottler: ts.svcAddrUpdateThrottler,
|
|
functionTimeoutMap: fnTimeoutMap,
|
|
unTapServiceTimeout: ts.unTapServiceTimeout,
|
|
}
|
|
|
|
internalRoute := utils.UrlForFunction(fn.ObjectMeta.Name, fn.ObjectMeta.Namespace)
|
|
internalPrefixRoute := internalRoute + "/"
|
|
handler := http.HandlerFunc(fh.handler)
|
|
muxRouter.Handle(internalRoute, handler)
|
|
muxRouter.PathPrefix(internalPrefixRoute).Handler(handler)
|
|
ts.logger.Debug("add internal handler and prefix route for function", zap.String("router", internalRoute), zap.Any("function", fn))
|
|
}
|
|
|
|
if featureConfig.AuthConfig.IsEnabled {
|
|
|
|
path := featureConfig.AuthConfig.AuthUriPath
|
|
// Auth endpoint for the router.
|
|
muxRouter.HandleFunc(path, authLoginHandler(featureConfig)).Methods("POST")
|
|
}
|
|
|
|
// Healthz endpoint for the router.
|
|
muxRouter.HandleFunc("/router-healthz", routerHealthHandler).Methods("GET")
|
|
// version of application.
|
|
muxRouter.HandleFunc("/_version", versionHandler).Methods("GET")
|
|
|
|
return muxRouter, nil
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *fv1.HTTPTrigger, err error) {
|
|
// TODO
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) addTriggerHandlers() error {
|
|
for _, triggerInformer := range ts.snapshotTriggerInformers() {
|
|
_, err := triggerInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) {
|
|
trigger := obj.(*fv1.HTTPTrigger)
|
|
go createIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
|
|
ts.syncTriggers()
|
|
},
|
|
DeleteFunc: func(obj interface{}) {
|
|
ts.syncTriggers()
|
|
trigger := obj.(*fv1.HTTPTrigger)
|
|
go deleteIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
|
|
},
|
|
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
|
oldTrigger := oldObj.(*fv1.HTTPTrigger)
|
|
newTrigger := newObj.(*fv1.HTTPTrigger)
|
|
|
|
if oldTrigger.ObjectMeta.ResourceVersion == newTrigger.ObjectMeta.ResourceVersion {
|
|
return
|
|
}
|
|
|
|
go updateIngress(context.Background(), ts.logger, oldTrigger, newTrigger, ts.kubeClient)
|
|
ts.syncTriggers()
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) addFunctionHandlers() error {
|
|
for _, funcInformer := range ts.snapshotFuncInformers() {
|
|
|
|
_, err := funcInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) {
|
|
ts.syncTriggers()
|
|
},
|
|
DeleteFunc: func(obj interface{}) {
|
|
ts.syncTriggers()
|
|
},
|
|
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
|
oldFn := oldObj.(*fv1.Function)
|
|
fn := newObj.(*fv1.Function)
|
|
|
|
if oldFn.ObjectMeta.ResourceVersion == fn.ObjectMeta.ResourceVersion {
|
|
return
|
|
}
|
|
|
|
// update resolver function reference cache
|
|
for key, rr := range ts.resolver.copy() {
|
|
if key.namespace == fn.ObjectMeta.Namespace &&
|
|
rr.functionMap[fn.ObjectMeta.Name] != nil &&
|
|
rr.functionMap[fn.ObjectMeta.Name].ObjectMeta.ResourceVersion != fn.ObjectMeta.ResourceVersion {
|
|
// invalidate resolver cache
|
|
ts.logger.Debug("invalidating resolver cache")
|
|
err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
|
|
if err != nil {
|
|
ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err))
|
|
}
|
|
break
|
|
}
|
|
}
|
|
ts.syncTriggers()
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) syncTriggers() {
|
|
ts.syncDebouncer(func() {
|
|
ts.updateRouterRequestChannel <- struct{}{}
|
|
})
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) snapshotTriggerInformers() []k8sCache.SharedIndexInformer {
|
|
ts.informerMu.RLock()
|
|
defer ts.informerMu.RUnlock()
|
|
|
|
informers := make([]k8sCache.SharedIndexInformer, 0, len(ts.triggerInformer))
|
|
for _, informer := range ts.triggerInformer {
|
|
informers = append(informers, informer)
|
|
}
|
|
return informers
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) snapshotFuncInformers() []k8sCache.SharedIndexInformer {
|
|
ts.informerMu.RLock()
|
|
defer ts.informerMu.RUnlock()
|
|
|
|
informers := make([]k8sCache.SharedIndexInformer, 0, len(ts.funcInformer))
|
|
for _, informer := range ts.funcInformer {
|
|
informers = append(informers, informer)
|
|
}
|
|
return informers
|
|
}
|
|
|
|
func (ts *HTTPTriggerSet) updateRouter(ctx context.Context) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ts.updateRouterRequestChannel:
|
|
}
|
|
// get triggers
|
|
alltriggers := make([]fv1.HTTPTrigger, 0)
|
|
for _, triggerInformer := range ts.snapshotTriggerInformers() {
|
|
latestTriggers := triggerInformer.GetStore().List()
|
|
for _, t := range latestTriggers {
|
|
alltriggers = append(alltriggers, *t.(*fv1.HTTPTrigger))
|
|
}
|
|
}
|
|
ts.triggers = alltriggers
|
|
|
|
// get functions
|
|
allfunctions := make([]fv1.Function, 0)
|
|
functionTimeout := make(map[types.UID]int, 0)
|
|
for _, funcInformer := range ts.snapshotFuncInformers() {
|
|
latestFunctions := funcInformer.GetStore().List()
|
|
for _, f := range latestFunctions {
|
|
fn := *f.(*fv1.Function)
|
|
functionTimeout[fn.ObjectMeta.UID] = fn.Spec.FunctionTimeout
|
|
allfunctions = append(allfunctions, fn)
|
|
}
|
|
}
|
|
ts.functions = allfunctions
|
|
|
|
// make a new router and use it
|
|
router, err := ts.getRouter(functionTimeout)
|
|
if err != nil {
|
|
ts.logger.Error("error updating router", zap.Error(err))
|
|
continue
|
|
}
|
|
ts.mutableRouter.updateRouter(router)
|
|
}
|
|
}
|
|
|
|
// AddNamespace dynamically registers a new namespace in the router without a restart.
|
|
// Creates per-NS informers for HTTPTriggers and Functions, wires up event handlers,
|
|
// and triggers a router rebuild. Safe to call repeatedly — deduplicates via NSResolver.
|
|
func (ts *HTTPTriggerSet) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
|
|
if !utils.DefaultNSResolver().AddNamespace(ns) {
|
|
return nil // already registered
|
|
}
|
|
ts.logger.Info("router.AddNamespace: setting up informers", zap.String("namespace", ns))
|
|
|
|
factory := genInformer.NewFilteredSharedInformerFactory(ts.fissionClient, 30*time.Minute, ns, nil)
|
|
triggerInf := factory.Core().V1().HTTPTriggers().Informer()
|
|
funcInf := factory.Core().V1().Functions().Informer()
|
|
|
|
_, err := triggerInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) {
|
|
trigger := obj.(*fv1.HTTPTrigger)
|
|
go createIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
|
|
ts.syncTriggers()
|
|
},
|
|
DeleteFunc: func(obj interface{}) {
|
|
ts.syncTriggers()
|
|
trigger := obj.(*fv1.HTTPTrigger)
|
|
go deleteIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
|
|
},
|
|
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
|
oldTrigger := oldObj.(*fv1.HTTPTrigger)
|
|
newTrigger := newObj.(*fv1.HTTPTrigger)
|
|
if oldTrigger.ObjectMeta.ResourceVersion == newTrigger.ObjectMeta.ResourceVersion {
|
|
return
|
|
}
|
|
go updateIngress(context.Background(), ts.logger, oldTrigger, newTrigger, ts.kubeClient)
|
|
ts.syncTriggers()
|
|
},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("router.AddNamespace %s: trigger handler: %w", ns, err)
|
|
}
|
|
|
|
_, err = funcInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) { ts.syncTriggers() },
|
|
DeleteFunc: func(obj interface{}) { ts.syncTriggers() },
|
|
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
|
oldFn := oldObj.(*fv1.Function)
|
|
fn := newObj.(*fv1.Function)
|
|
if oldFn.ObjectMeta.ResourceVersion == fn.ObjectMeta.ResourceVersion {
|
|
return
|
|
}
|
|
for key, rr := range ts.resolver.copy() {
|
|
if key.namespace == fn.ObjectMeta.Namespace &&
|
|
rr.functionMap[fn.ObjectMeta.Name] != nil &&
|
|
rr.functionMap[fn.ObjectMeta.Name].ObjectMeta.ResourceVersion != fn.ObjectMeta.ResourceVersion {
|
|
ts.logger.Debug("invalidating resolver cache")
|
|
_ = ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
|
|
break
|
|
}
|
|
}
|
|
ts.syncTriggers()
|
|
},
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("router.AddNamespace %s: func handler: %w", ns, err)
|
|
}
|
|
|
|
ts.informerMu.Lock()
|
|
ts.triggerInformer[ns] = triggerInf
|
|
ts.funcInformer[ns] = funcInf
|
|
ts.informerMu.Unlock()
|
|
ts.resolver.addInformer(ns, funcInf)
|
|
|
|
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
|
|
ns + "/trigger": triggerInf,
|
|
ns + "/func": funcInf,
|
|
})
|
|
factory.Start(ctx.Done())
|
|
// Wait for cache to sync before rebuilding the router, so triggers are visible.
|
|
k8sCache.WaitForCacheSync(ctx.Done(), triggerInf.HasSynced, funcInf.HasSynced)
|
|
|
|
ts.logger.Info("router.AddNamespace: done", zap.String("namespace", ns))
|
|
ts.syncTriggers()
|
|
return nil
|
|
}
|