diff --git a/docs/generator-parameters.md b/docs/generator-parameters.md index 7ce0b583..c639389c 100644 --- a/docs/generator-parameters.md +++ b/docs/generator-parameters.md @@ -113,6 +113,7 @@ For each microservice, HydraGen supports a set of configuration parameters that * **execution_mode**: Determines if the server responding at this endpoint should handle requests sequentially or in parallel, on multiple threads. Default: "sequential" * **cpu_complexity**: CPU stress parameters. * **network_complexity**: Network stress parameters. +* **resilience_patterns**: Resilience patterns parameters. #### Format @@ -121,7 +122,7 @@ For each microservice, HydraGen supports a set of configuration parameters that { "name": "", "execution_mode": "", - + "resilience_patterns": {...}, "cpu_complexity": {...}, "network_complexity": {...} }, @@ -129,6 +130,32 @@ For each microservice, HydraGen supports a set of configuration parameters that ] ``` +## Describing Resilience Patterns + +Hydragen supports the injection of resilience patterns into the client generated code. Initially, Circuit Breaker pattern is supported. + +### Circuit Breaker + +The circuit breaker implementation use a simple strategy to change the circuit breaker state (_CLOSED, OPEN and HALF OPEN_). If some request exceeds the timeout, the circuit breaker implementation will change the state to _OPEN_ (This meansthat the failed request threshold it's one request). After a configured timeout, the pattern will send a request to the destination service and check the response status. + +The circuit breaker is configured for the hole endpoint, but it must be enabled for each called service in Network stressor. + +#### Required attributes + +* **timeout**: Determines the timeout in seconds to consider the response as failed. +* **retry_timer**: Determines the number of seconds that the circuit breaker must wait to retry a request to the destination service connection. + +#### Format + +```json +"resilience_patterns" : { + "circuit_breaker": { + "timeout": , + "retry_timer: + } +} +``` + ## Describing Resource Stressors HydraGen supports parameters to express the computational complexity or stress a microservice exerts on the different hardware resources. Initially, CPU-bounded or network-bounded tasks are implemented. The complexity of a CPU-bounded task can be described based on the time a busy-wait is executed, while the load on the network I/O can be described by specifying parameters such as the call forwarding mode and the request/response size for each service endpoint call. @@ -168,7 +195,6 @@ The CPU stressor will lock threads for exclusive access while it is executing. T "network_complexity": { "forward_requests": "", "response_payload_size": , - "called_services": [...] } ``` @@ -186,6 +212,7 @@ The CPU stressor will lock threads for exclusive access while it is executing. T * **request_payload_size**: Determines the number of characters that will be sent in the request to the endpoint. Default: 0 * **port**: The port the server is responding to requests on. This is usually determined automatically. * **protocol**: Determines if the call will be made using HTTP or gRPC. This is usually determined automatically. +* **active_circuit_breaker**: Determines if the endpoint circuit breaker will protect this service call. #### Format @@ -197,7 +224,8 @@ The CPU stressor will lock threads for exclusive access while it is executing. T "port": "", "protocol": "", "traffic_forward_ratio": , - "request_payload_size": + "request_payload_size": , + "active_circuit_breaker": } ] ``` diff --git a/emulator/src/client/grpc.go b/emulator/src/client/grpc.go index 77ac10e8..78d73b5e 100644 --- a/emulator/src/client/grpc.go +++ b/emulator/src/client/grpc.go @@ -18,6 +18,7 @@ package client import ( "application-emulator/src/generated/client" + "application-emulator/src/resilience/circuit_breaker" "application-model/generated" "context" "fmt" @@ -28,7 +29,7 @@ import ( ) // Sends a gRPC request to the specified endpoint -func GRPC(service, endpoint string, port int, payload string) (*generated.Response, error) { +func GRPC(service, endpoint string, port int, payload, sourceEndpoint string) (*generated.Response, error) { var url string // Omit the port if zero if port == 0 { @@ -46,11 +47,25 @@ func GRPC(service, endpoint string, port int, payload string) (*generated.Respon } defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() + circuitBreakerRegistry := circuit_breaker.GetCircuitBreakerRegistry() + cbName := circuitBreakerRegistry.BuildName(sourceEndpoint, service, endpoint) + circuitBreaker := circuitBreakerRegistry.GetCircuitBreaker(cbName) + var response *generated.Response + request := &generated.Request{ + Payload: payload, + } callOptions := []grpc.CallOption{} - response, err := client.CallGeneratedEndpoint(ctx, conn, service, endpoint, &generated.Request{Payload: payload}, callOptions...) + + if circuitBreaker == nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + response, err = client.CallGeneratedEndpoint(ctx, conn, service, endpoint, request, callOptions...) + } else { + response, err = circuitBreaker.ProxyGRPC(conn, service, endpoint, request, callOptions...) + } + if err != nil { return nil, err } diff --git a/emulator/src/client/restful.go b/emulator/src/client/restful.go index ff1e2bc6..afd23dec 100644 --- a/emulator/src/client/restful.go +++ b/emulator/src/client/restful.go @@ -17,11 +17,13 @@ limitations under the License. package client import ( + "application-emulator/src/resilience/circuit_breaker" "application-model/generated" "bytes" "encoding/json" "fmt" "io" + "log" "net/http" "google.golang.org/protobuf/encoding/protojson" @@ -30,7 +32,7 @@ import ( const useProtoJSON = true // Sends a HTTP POST request to the specified endpoint -func POST(service, endpoint string, port int, payload string, headers http.Header) (int, *generated.Response, error) { +func POST(service, endpoint string, port int, payload string, headers http.Header, sourceEndpoint string) (int, *generated.Response, error) { var url string // Omit the port if zero if port == 0 { @@ -58,7 +60,24 @@ func POST(service, endpoint string, port int, payload string, headers http.Heade } // Send the request - response, err := http.DefaultClient.Do(request) + // Here it comes the circuit breaker + + circuitBreakerRegistry := circuit_breaker.GetCircuitBreakerRegistry() + cbName := circuitBreakerRegistry.BuildName(sourceEndpoint, service, endpoint) + circuitBreaker := circuitBreakerRegistry.GetCircuitBreaker(cbName) + + log.Printf("[CLIENT HTTP] Circuit breaker obtanied %v", circuitBreaker) + + var response *http.Response + var err error + + if circuitBreaker == nil { + response, err = http.DefaultClient.Do(request) + } else { + response, err = circuitBreaker.ProxyHTTP(request) + } + + log.Printf("[CLIENT HTTP] Request returned from %s/%s", service, endpoint) if err != nil { return 0, nil, err } diff --git a/emulator/src/resilience/circuit_breaker/implementation.go b/emulator/src/resilience/circuit_breaker/implementation.go new file mode 100644 index 00000000..2c0dcfba --- /dev/null +++ b/emulator/src/resilience/circuit_breaker/implementation.go @@ -0,0 +1,122 @@ +package circuit_breaker + +import ( + "application-emulator/src/generated/client" + "application-model/generated" + "context" + "errors" + "log" + "net/http" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type CircuitBreakerState string + +const ( + OPEN CircuitBreakerState = "OPEN" + CLOSED CircuitBreakerState = "CLOSED" + HALF_OPEN CircuitBreakerState = "HALF OPEN" +) + +var GRPC_ERROR = status.Error(codes.Unavailable, "Service unavailable") +var HTTP_ERROR = errors.New("Service unavailable") + +type RequestCallback func(ctx context.Context) (any, error) + +type CircuitBreaker interface { + ProxyHTTP(request *http.Request) (*http.Response, error) + ProxyGRPC(conn *grpc.ClientConn, service, endpoint string, request *generated.Request, options ...grpc.CallOption) (*generated.Response, error) + ProcessRequest(cb RequestCallback, requestError error) (any, error) +} + +type CircuitBreakerImpl struct { + State CircuitBreakerState + Timeout int // timeout in seconds + RetryTimer int // In how many seconds should we retry + lock sync.Mutex + EndpointProtected string +} + +func (c *CircuitBreakerImpl) ProcessRequest(cb RequestCallback, requestError error) (any, error) { + log.Printf("[CIRCUIT BREAKER] Circuit breaker of %s in state %s\n", c.EndpointProtected, c.State) + c.lock.Lock() + if c.State == OPEN { + c.lock.Unlock() + return nil, requestError + } + if c.State == HALF_OPEN { + c.State = OPEN + } + c.lock.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.Timeout)*time.Second) + defer cancel() + + response, err := cb(ctx) + log.Printf("[CIRCUIT BREAKER] Request sended and callback returned %v, with error %v\n", response, err) + + if err != nil && errors.Is(err, context.DeadlineExceeded) { + log.Printf("[CIRCUIT BREAKER] Circuit breaker of %s timedout\n", c.EndpointProtected) + c.lock.Lock() + c.State = OPEN + c.lock.Unlock() + go func() { + time.Sleep(time.Second * time.Duration(c.RetryTimer)) + c.lock.Lock() + c.State = HALF_OPEN + c.lock.Unlock() + return + }() + return nil, requestError + } + + c.lock.Lock() + if c.State == OPEN || c.State == HALF_OPEN { + c.State = CLOSED + } + c.lock.Unlock() + return response, err +} + +func (c *CircuitBreakerImpl) ProxyHTTP(request *http.Request) (*http.Response, error) { + + response, err := c.ProcessRequest(func(ctx context.Context) (any, error) { + request = request.WithContext(ctx) + response, err := http.DefaultClient.Do(request) + return response, err + }, HTTP_ERROR) + + if err != nil { + return nil, err + } + + httpResponse, ok := response.(*http.Response) + log.Printf("[CIRCUIT BREAKER] The response returned from circuit breaker as %v", httpResponse) + + if !ok { + return nil, errors.New("HTTP response from Circuit breaker callback broken") + } + + return httpResponse, err +} + +func (c *CircuitBreakerImpl) ProxyGRPC(conn *grpc.ClientConn, service, endpoint string, request *generated.Request, options ...grpc.CallOption) (*generated.Response, error) { + response, err := c.ProcessRequest(func(ctx context.Context) (any, error) { + return client.CallGeneratedEndpoint(ctx, conn, service, endpoint, request, options...) + }, GRPC_ERROR) + + if err != nil { + return nil, err + } + grpcResponse, ok := response.(*generated.Response) + if !ok { + return nil, errors.New("GRPC response from Circuit breaker callback broken") + } + + return grpcResponse, err +} diff --git a/emulator/src/resilience/circuit_breaker/register.go b/emulator/src/resilience/circuit_breaker/register.go new file mode 100644 index 00000000..378a920a --- /dev/null +++ b/emulator/src/resilience/circuit_breaker/register.go @@ -0,0 +1,59 @@ +package circuit_breaker + +import ( + model "application-model" + "fmt" + "sync" +) + +var instanceLock = &sync.Mutex{} +var registerLock = &sync.Mutex{} +var GetLock = &sync.Mutex{} + +type CircuitBreakerRegister struct { + ProtectedEndpoints map[string]*CircuitBreakerImpl +} + +var circuitBreakerInstance *CircuitBreakerRegister + +func (cbr *CircuitBreakerRegister) BuildName(sourceEndpoint, destService, destEndpoint string) string { + return fmt.Sprintf("%s:%s/%s", sourceEndpoint, destService, destEndpoint) +} +func (cbr *CircuitBreakerRegister) RegisterEndpoint(endpoint string, config *model.CircuitBreakerConfig) { + registerLock.Lock() + defer registerLock.Unlock() + cbr.ProtectedEndpoints[endpoint] = &CircuitBreakerImpl{ + State: CLOSED, + Timeout: config.Timeout, + RetryTimer: config.RetryTimer, + EndpointProtected: endpoint, + } +} + +func (cbr *CircuitBreakerRegister) GetCircuitBreaker(endpoint string) *CircuitBreakerImpl { + GetLock.Lock() + defer GetLock.Unlock() + + circuitBreaker, ok := cbr.ProtectedEndpoints[endpoint] + if !ok { + return nil + } + return circuitBreaker +} + +func CheckCircuitBreakerConfig(endpoint *model.Endpoint) bool { + return endpoint.ResiliencePatterns.CircuitBreaker != nil +} + +func GetCircuitBreakerRegistry() *CircuitBreakerRegister { + if circuitBreakerInstance == nil { + instanceLock.Lock() + defer instanceLock.Unlock() + if circuitBreakerInstance == nil { + circuitBreakerInstance = &CircuitBreakerRegister{ + ProtectedEndpoints: make(map[string]*CircuitBreakerImpl), + } + } + } + return circuitBreakerInstance +} diff --git a/emulator/src/stressors/forward.go b/emulator/src/stressors/forward.go index b8cc3024..aae1adcc 100644 --- a/emulator/src/stressors/forward.go +++ b/emulator/src/stressors/forward.go @@ -21,6 +21,7 @@ import ( model "application-model" "application-model/generated" "fmt" + "log" "net/http" "sync" @@ -53,10 +54,12 @@ func ExtractHeaders(request any) http.Header { return forwardHeaders } -func httpRequest(service model.CalledService, forwardHeaders http.Header) generated.EndpointResponse { +func httpRequest(service model.CalledService, forwardHeaders http.Header, sourceEndpoint string) generated.EndpointResponse { + log.Printf("[FORWARD] Http request ready to send POST to %s/%s", service.Service, service.Endpoint) status, response, err := - client.POST(service.Service, service.Endpoint, service.Port, RandomPayload(service.RequestPayloadSize), forwardHeaders) + client.POST(service.Service, service.Endpoint, service.Port, RandomPayload(service.RequestPayloadSize), forwardHeaders, sourceEndpoint) + log.Printf("[FORWARD] Http request returned from %s/%s", service.Service, service.Endpoint) if err != nil { return generated.EndpointResponse{ Service: &service, @@ -73,9 +76,9 @@ func httpRequest(service model.CalledService, forwardHeaders http.Header) genera } } -func grpcRequest(service model.CalledService) generated.EndpointResponse { +func grpcRequest(service model.CalledService, sourceEndpoint string) generated.EndpointResponse { response, err := - client.GRPC(service.Service, service.Endpoint, service.Port, RandomPayload(service.RequestPayloadSize)) + client.GRPC(service.Service, service.Endpoint, service.Port, RandomPayload(service.RequestPayloadSize), sourceEndpoint) if err != nil { return generated.EndpointResponse{ @@ -94,7 +97,7 @@ func grpcRequest(service model.CalledService) generated.EndpointResponse { } // Forward requests to all services sequentially and return REST or gRPC responses -func ForwardSequential(request any, services []model.CalledService) []generated.EndpointResponse { +func ForwardSequential(request any, services []model.CalledService, endpoint string) []generated.EndpointResponse { forwardHeaders := ExtractHeaders(request) len := 0 for _, service := range services { @@ -106,10 +109,10 @@ func ForwardSequential(request any, services []model.CalledService) []generated. for _, service := range services { for j := 0; j < service.TrafficForwardRatio; j++ { if service.Protocol == "http" { - response := httpRequest(service, forwardHeaders) + response := httpRequest(service, forwardHeaders, endpoint) responses[i] = response } else if service.Protocol == "grpc" { - response := grpcRequest(service) + response := grpcRequest(service, endpoint) responses[i] = response } i++ @@ -119,22 +122,22 @@ func ForwardSequential(request any, services []model.CalledService) []generated. return responses } -func parallelHTTPRequest(responses []generated.EndpointResponse, i int, service model.CalledService, forwardHeaders http.Header, wg *sync.WaitGroup) { +func parallelHTTPRequest(responses []generated.EndpointResponse, i int, service model.CalledService, forwardHeaders http.Header, wg *sync.WaitGroup, endpoint string) { defer wg.Done() - response := httpRequest(service, forwardHeaders) + response := httpRequest(service, forwardHeaders, endpoint) // No mutex needed since every response has its own index responses[i] = response } -func parallelGRPCRequest(responses []generated.EndpointResponse, i int, service model.CalledService, wg *sync.WaitGroup) { +func parallelGRPCRequest(responses []generated.EndpointResponse, i int, service model.CalledService, wg *sync.WaitGroup, endpoint string) { defer wg.Done() - response := grpcRequest(service) + response := grpcRequest(service, endpoint) // No mutex needed since every response has its own index responses[i] = response } // Forward requests to all services in parallel using goroutines and return REST or gRPC responses -func ForwardParallel(request any, services []model.CalledService) []generated.EndpointResponse { +func ForwardParallel(request any, services []model.CalledService, endpoint string) []generated.EndpointResponse { forwardHeaders := ExtractHeaders(request) len := 0 for _, service := range services { @@ -148,10 +151,10 @@ func ForwardParallel(request any, services []model.CalledService) []generated.En for j := 0; j < service.TrafficForwardRatio; j++ { if service.Protocol == "http" { wg.Add(1) - go parallelHTTPRequest(responses, i, service, forwardHeaders, &wg) + go parallelHTTPRequest(responses, i, service, forwardHeaders, &wg, endpoint) } else if service.Protocol == "grpc" { wg.Add(1) - go parallelGRPCRequest(responses, i, service, &wg) + go parallelGRPCRequest(responses, i, service, &wg, endpoint) } i++ } diff --git a/emulator/src/stressors/network.go b/emulator/src/stressors/network.go index 7ffae0b1..d1028a22 100644 --- a/emulator/src/stressors/network.go +++ b/emulator/src/stressors/network.go @@ -17,10 +17,12 @@ limitations under the License. package stressors import ( + "application-emulator/src/resilience/circuit_breaker" "application-emulator/src/util" model "application-model" "application-model/generated" "fmt" + "log" "math/rand" "strings" ) @@ -110,11 +112,26 @@ func (n *NetworkTask) ExecAllowed(endpoint *model.Endpoint) bool { func (n *NetworkTask) ExecTask(endpoint *model.Endpoint, responses *MutexTaskResponses) { stressParams := endpoint.NetworkComplexity + log.Printf("[NETWORK STRESSOR] Recieved request to endpoint %s", endpoint.Name) + + circuitBreakerRegistry := circuit_breaker.GetCircuitBreakerRegistry() + + for _, svc := range stressParams.CalledServices { + if svc.ActiveCircuitBreaker { + cbName := circuitBreakerRegistry.BuildName(endpoint.Name, svc.Service, svc.Endpoint) + cb := circuitBreakerRegistry.GetCircuitBreaker(cbName) + if cb == nil { + circuitBreakerRegistry.RegisterEndpoint(cbName, endpoint.ResiliencePatterns.CircuitBreaker) + } + } + } + + log.Printf("[NETWORK STRESSOR] Circuit Breaker verified!!") var calls []generated.EndpointResponse if stressParams.ForwardRequests == "asynchronous" { - calls = ForwardParallel(n.Request, stressParams.CalledServices) + calls = ForwardParallel(n.Request, stressParams.CalledServices, endpoint.Name) } else if stressParams.ForwardRequests == "synchronous" { - calls = ForwardSequential(n.Request, stressParams.CalledServices) + calls = ForwardSequential(n.Request, stressParams.CalledServices, endpoint.Name) } svc := fmt.Sprintf("%s/%s", util.ServiceName, endpoint.Name) diff --git a/generator/elk/elasticsearch_statefulset.yaml b/generator/elk/elasticsearch_statefulset.yaml index bfa8d59d..cbdbd04a 100644 --- a/generator/elk/elasticsearch_statefulset.yaml +++ b/generator/elk/elasticsearch_statefulset.yaml @@ -82,4 +82,4 @@ spec: volumes: - name: data persistentVolumeClaim: - claimName: elk-pvc \ No newline at end of file + claimName: elk-pvc diff --git a/generator/elk/elasticsearch_svc.yaml b/generator/elk/elasticsearch_svc.yaml index 29843459..97188387 100644 --- a/generator/elk/elasticsearch_svc.yaml +++ b/generator/elk/elasticsearch_svc.yaml @@ -29,4 +29,4 @@ spec: nodePort: 30001 name: rest - port: 9300 - name: inter-node \ No newline at end of file + name: inter-node diff --git a/generator/elk/fluentd.yaml b/generator/elk/fluentd.yaml index 3f26ee38..f2e19c98 100644 --- a/generator/elk/fluentd.yaml +++ b/generator/elk/fluentd.yaml @@ -107,4 +107,4 @@ spec: path: /var/log - name: varlibdockercontainers hostPath: - path: /var/lib/docker/containers \ No newline at end of file + path: /var/lib/docker/containers diff --git a/generator/elk/kibana.yaml b/generator/elk/kibana.yaml index e7efea21..2ff145ab 100644 --- a/generator/elk/kibana.yaml +++ b/generator/elk/kibana.yaml @@ -57,4 +57,4 @@ spec: - name: ELASTICSEARCH_URL value: http://elasticsearch:9200 ports: - - containerPort: 5601 \ No newline at end of file + - containerPort: 5601 diff --git a/generator/elk/pv.yaml b/generator/elk/pv.yaml index 42a066a3..a3e39024 100644 --- a/generator/elk/pv.yaml +++ b/generator/elk/pv.yaml @@ -28,4 +28,4 @@ spec: name: elk-pvc namespace: logging hostPath: - path: "/var/log/elk/" \ No newline at end of file + path: "/var/log/elk/" diff --git a/generator/elk/pvc.yaml b/generator/elk/pvc.yaml index e70ca8a7..8b2ba924 100644 --- a/generator/elk/pvc.yaml +++ b/generator/elk/pvc.yaml @@ -25,4 +25,4 @@ spec: resources: requests: storage: 10Gi - volumeName: elk-pv \ No newline at end of file + volumeName: elk-pv diff --git a/generator/src/pkg/generate/generate.go b/generator/src/pkg/generate/generate.go index 84ea6345..3ed3bc06 100644 --- a/generator/src/pkg/generate/generate.go +++ b/generator/src/pkg/generate/generate.go @@ -346,7 +346,7 @@ func CreateDockerImage(config model.FileConfig, buildHash string) { "--build-arg", "SRCIMAGE=" + sourceImage, "--build-arg", - "BASEIMAGE=" + config.Settings.BaseImage, + "BASEIMAGE=" + config.Settings.BaseImage, // busybox by default path, } diff --git a/generator/src/pkg/generate/validation.go b/generator/src/pkg/generate/validation.go index 3efc10dc..2b0a4235 100644 --- a/generator/src/pkg/generate/validation.go +++ b/generator/src/pkg/generate/validation.go @@ -129,6 +129,20 @@ func ValidateProtocols(service *model.Service) error { } for _, endpoint := range service.Endpoints { + + // Circuit breaker validations + if endpoint.ResiliencePatterns != nil { + if endpoint.NetworkComplexity == nil { + return fmt.Errorf("Resilience must have network calls to protect") + } + + if endpoint.ResiliencePatterns.CircuitBreaker != nil { + if endpoint.ResiliencePatterns.CircuitBreaker.RetryTimer == 0 || endpoint.ResiliencePatterns.CircuitBreaker.Timeout == 0 { + return fmt.Errorf("Circuit breaker must have timeout and retry timer values > 0") + } + } + } + if endpoint.NetworkComplexity != nil { for _, calledService := range endpoint.NetworkComplexity.CalledServices { if !validProtocols[calledService.Protocol] { diff --git a/model/input.go b/model/input.go index 04b9b690..d41277c4 100644 --- a/model/input.go +++ b/model/input.go @@ -17,12 +17,13 @@ limitations under the License. package model type CalledService struct { - Service string `json:"service"` - Port int `json:"port"` - Endpoint string `json:"endpoint"` - Protocol string `json:"protocol"` - TrafficForwardRatio int `json:"traffic_forward_ratio"` - RequestPayloadSize int `json:"request_payload_size"` + Service string `json:"service"` + Port int `json:"port"` + Endpoint string `json:"endpoint"` + Protocol string `json:"protocol"` + TrafficForwardRatio int `json:"traffic_forward_ratio"` + RequestPayloadSize int `json:"request_payload_size"` + ActiveCircuitBreaker bool `json:"active_circuit_breaker"` } type CpuComplexity struct { @@ -36,11 +37,22 @@ type NetworkComplexity struct { CalledServices []CalledService `json:"called_services"` } +type CircuitBreakerConfig struct { + Timeout int `json:"timeout"` + RetryTimer int `json:"retry_timer"` +} + +// TODO: Implement more Resilience patterns +type ResiliencePatterns struct { + CircuitBreaker *CircuitBreakerConfig `json:"circuit_breaker,omitempty"` +} + type Endpoint struct { - Name string `json:"name"` - ExecutionMode string `json:"execution_mode"` - CpuComplexity *CpuComplexity `json:"cpu_complexity,omitempty"` - NetworkComplexity *NetworkComplexity `json:"network_complexity,omitempty"` + Name string `json:"name"` + ExecutionMode string `json:"execution_mode"` + CpuComplexity *CpuComplexity `json:"cpu_complexity,omitempty"` + NetworkComplexity *NetworkComplexity `json:"network_complexity,omitempty"` + ResiliencePatterns *ResiliencePatterns `json:"resilience_patterns,omitempty"` } type ResourceLimits struct {