---
fault:
  - entity:
      name: load-generator-pod-1
      group_id: load-generator-pod-1
      kind: Pod
    condition: To be specified
    category: Change
    fault_mechanism: custom
alerts:
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Error rate is above threshold for frontend-proxy service
  - id: RequestLatency
    group_id: frontend-proxy-service-1
    metadata:
      description: Latency is above threshold for frontend-proxy service
groups:
  - id: load-generator-pod-1
    kind: Pod
    filter:
      - load-generator-.*
    namespace: otel-demo
    root_cause: true
  - id: load-generator-service-1
    kind: Service
    filter:
      - load-generator\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-proxy-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - frontend-proxy-.*
aliases:
  - - load-generator-service-1
    - load-generator-pod-1
    - frontend-proxy-service-1
    - frontend-proxy-pod-1
propagations:
  - source: load-generator-pod-1
    target: load-generator-service-1
    condition: load-generator pod is configured with a higher number of users
    effect: load-generator creates a high number of requests as a result
  - source: load-generator-service-1
    target: frontend-proxy-service-1
    condition: frontend-proxy service overloaded - request volume exceeds configured capacity
    effect: frontend-proxy service error rate is above threshold
  - source: load-generator-service-1
    target: frontend-proxy-service-1
    condition: frontend-proxy service overloaded - request volume exceeds configured capacity
    effect: frontend-proxy service latency is above threshold
recommended_actions:
  - solution:
      id: deem_if_traffic_is_legit
      actions:
        - deem if traffic is legit
  - solution:
      id: set_up_autoscaler
      actions:
        - set up autoscaler


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High HTTP error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: cart
      kind: Service
    condition: High HTTP error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
    changed:
      element: "data.demo\\.flagd\\.json"
      from: "\"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    }"
      to: "\"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    }"
    condition: cartFailure feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: cart-service-1
    kind: Service
    filter:
      - cart\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
alerts:
  - id: RequestErrorRate
    group_id: cart-service-1
    metadata:
      description: Cart service EmptyCart operations failing due to feature flag
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors when users attempt to empty cart
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing errors from failed cart operations
propagations:
  - source: flagd-config
    target: cart-service-1
    condition: Feature flag causes cart service EmptyCart method to fail
    effect: Cart service returns errors when users attempt to empty their shopping cart
  - source: cart-service-1
    target: frontend-service-1
    condition: Frontend receives errors from cart service when users try to empty cart
    effect: Users cannot empty their shopping carts, affecting checkout flow
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns error responses to proxy for failed cart operations
    effect: Users experience cart functionality failures through the proxy
recommended_actions:
  - solution:
      id: Disable cartFailure flag
      actions:
        - Disable cartFailure flag
  - solution:
      id: Restart cart service
      actions:
        - Restart cart service


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-3
spec:
  alerts:
    - group_id: frontend-service-1
      id: HighRequestErrorRate
      metadata:
        description: Frontend experiencing errors when calling ad service with high CPU
    - group_id: frontend-proxy-service-1
      id: HighRequestErrorRate
      metadata:
        description: Frontend-proxy experiencing cascading errors from frontend failures
  aliases:
    - - flagd-pod-1
      - flagd-service-1
    - - frontend-service-1
      - frontend-pod-1
    - - frontend-proxy-service-1
    - - ad-service-1
      - ad-pod-1
  fault:
    - category: Resource Saturation
      condition: High HTTP error rate
      entity:
        kind: Service
        name: frontend
      fault_mechanism: custom(flagd)
    - category: Resource Saturation
      condition: High HTTP error rate
      entity:
        kind: Service
        name: cart
      fault_mechanism: custom(flagd)
    - category: Configuration Setting
      condition: adHighCpu feature flag set
      entity:
        changed:
          element: data.demo\.flagd\.json
          from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product
            catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n\
            \      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n   \
            \   \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n\
            \    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"\
            variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adHighCpu\": {\n      \"description\"\
            : \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n\
            \      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\"\
            ,\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\"\
            : {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\"\
            : \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\"\
            : {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n\
            \      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests n%\",\n\
            \      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"100%\": 1,\n        \"90%\": 0.95,\n        \"75%\": 0.75,\n        \"50%\": 0.5,\n \
            \       \"25%\": 0.25,\n        \"10%\": 0.1,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n\
            \      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\"\
            : false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadGeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with
            a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\"\
            : \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"\
            variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
          to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product
            catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n\
            \      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n   \
            \   \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n\
            \    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"\
            variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adHighCpu\": {\n      \"description\"\
            : \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n\
            \      },\n      \"defaultVariant\": \"on\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n\
            \      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\"\
            : {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\"\
            : \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\"\
            : {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n\
            \      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests n%\",\n\
            \      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"100%\": 1,\n        \"90%\": 0.95,\n        \"75%\": 0.75,\n        \"50%\": 0.5,\n \
            \       \"25%\": 0.25,\n        \"10%\": 0.1,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n\
            \      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\"\
            : false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadGeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with
            a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\"\
            : \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"\
            variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        kind: ConfigMap
        name: flagd-config
      fault_mechanism: custom(flagd)
  groups:
    - filter:
        - flagd-config\b
      id: flagd-config-1
      kind: ConfigMap
      namespace: otel-demo
      root_cause: true
    - filter:
        - flagd-.*
      id: flagd-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - flagd\b
      id: flagd-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - adservice-.*
      id: ad-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - adservice\b
      id: ad-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend\b
      id: frontend-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-.*
      id: frontend-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend-proxy\b
      id: frontend-proxy-service-1
      kind: Service
      namespace: otel-demo
  propagations:
    - condition: flagd-config ConfigMap adHighCpu feature flag set
      effect: flagd enables injection
      source: flagd-config-1
      target: flagd-service-1
    - condition: flagd enables injection
      effect: ad pod High CPU Utilization. ad service Error Rate is Above Threshold
      source: flagd-service-1
      target: ad-service-1
    - condition: ad service is returning errors
      effect: frontend Error Rate is Above Threshold
      source: ad-service-1
      target: frontend-service-1
    - condition: frontend experiencing errors from ad service
      effect: frontend-proxy experiences cascading errors
      source: frontend-service-1
      target: frontend-proxy-service-1
  recommendedActions:
    - solution:
        actions:
          - Disable adHighCpu feature flag
        id: Disable_adHighCpu_feature_flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: "data.demo\\.flagd\\.json"
        from: '"productCatalogFailure": { "defaultVariant": "off" }'
        to: '"productCatalogFailure": { "defaultVariant": "on" }'
    condition: productCatalogFailure feature flag enabled
    category: Configuration Setting
    fault_mechanism: custom(flagd)
alerts:
  - id: RequestErrorRate
    group_id: product-catalog-service-1
    metadata:
      description: Request error rate in service product-catalog in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Request error rate in service checkout in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Request error rate in service frontend in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Request error rate in service frontend-proxy in namespace otel-demo is above 0
groups:
  - id: flagd-config-1
    kind: ConfigMap
    namespace: otel-demo
    filter:
      - flagd-config\b
    root_cause: true
  - id: flagd-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - flagd-.*
  - id: flagd-service-1
    kind: Service
    namespace: otel-demo
    filter:
      - flagd\b
  - id: product-catalog-service-1
    kind: Service
    filter:
      - product-catalog\b
    namespace: otel-demo
  - id: product-catalog-pod-1
    kind: Pod
    filter:
      - product-catalog-.*
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: checkout-pod-1
    kind: Pod
    filter:
      - checkout-.*
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    filter:
      - frontend-.*
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-proxy-pod-1
    kind: Pod
    filter:
      - frontend-proxy-.*
    namespace: otel-demo
aliases:
  - - flagd-pod-1
    - flagd-service-1
  - - product-catalog-service-1
    - product-catalog-pod-1
  - - checkout-service-1
    - checkout-pod-1
  - - frontend-service-1
    - frontend-pod-1
  - - frontend-proxy-service-1
    - frontend-proxy-pod-1
propagations:
  - source: flagd-config-1
    target: flagd-service-1
    condition: flagd-config ConfigMap productCatalogFailure feature flag enabled
    effect: flagd enables product catalog failure injection
  - source: flagd-service-1
    target: product-catalog-service-1
    condition: flagd enables product catalog failure injection
    effect: product-catalog service returns errors for specific product requests
  - source: product-catalog-service-1
    target: checkout-service-1
    condition: checkout service receives errors when retrieving product information
    effect: checkout service experiences errors during product validation
  - source: checkout-service-1
    target: frontend-service-1
    condition: frontend receives errors from checkout service due to product catalog failures
    effect: frontend service experiences errors during checkout operations
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: frontend-proxy receives errors from frontend service due to checkout failures
    effect: frontend-proxy service experiences errors when proxying requests
recommended_actions:
  - solution:
      id: disable_productCatalogFailure_feature_flag
      actions:
        - Disable productCatalogFailure feature flag in flagd configuration


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: adservice
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
    changed:
      element: "data.demo\\.flagd\\.json"
      from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
      to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: adFailure feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
alerts:
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Error Rate is Above Threshold
groups:
  - id: flagd-config-1
    kind: ConfigMap
    namespace: otel-demo
    filter:
      - flagd-config
    root_cause: true
  - id: flagd-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - flagd-.*
  - id: flagd-service-1
    kind: Service
    namespace: otel-demo
    filter:
      - flagd
  - id: getads-endpoint-1
    kind: ServiceOperation
    namespace: otel-demo
    filter:
      - ad-.*_AdService_GetAds
  - id: ad-service-1
    kind: Service
    filter:
      - ad
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
        - frontend
    namespace: otel-demo
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - frontend-.*
aliases:
  - - flagd-pod-1
    - flagd-service-1
  - - ad-service-1
    - ad-pod-1
    - getads-endpoint-1
  - - frontend-service-1
    - frontend-pod-1
propagations:
  - source: flagd-config-1
    target: flagd-service-1
    condition: flagd-config ConfigMap adFailure feature flag set
    effect: flagd enables injection
  - source: flagd-service-1
    target: getads-endpoint-1
    condition: flagd enables injection
    effect: getads endpoint Error Rate is Above Threshold
  - source: getads-endpoint-1
    target: ad-service-1
    condition: flagd-config ConfigMap adFailure feature flag set
    effect: ad service Error Rate is Above Threshold
  - source: ad-service-1
    target: frontend-service-1
    condition: ad service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
recommended_actions:
  - solution:
      id: Disable_adFailure_feature_flag
      actions:
        - Disable adFailure feature flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: ad
      kind: Service
    condition: High latency
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: frontend
      kind: Service
    condition: High error rate and high latency
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
    changed:
      element: "data.demo\\.flagd\\.json"
      from: "\"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    }"
      to: "\"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    }"
    condition: adManualGc feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: ad-service-1
    kind: Service
    filter:
      - ad\b
    namespace: otel-demo
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    filter:
      - frontend-.*
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-proxy-pod-1
    kind: Pod
    filter:
      - frontend-proxy-.*
    namespace: otel-demo
aliases:
  - - ad-service-1
    - ad-pod-1
  - - frontend-service-1
    - frontend-pod-1
  - - frontend-proxy-service-1
    - frontend-proxy-pod-1
alerts:
  - id: RequestLatency
    group_id: ad-service-1
    metadata:
      description: Ad service experiencing increased response times due to GC pauses
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing timeouts or errors when calling ad service
  - id: RequestLatency
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing increased latency waiting for ad service responses
  - id: RequestLatency
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing increased latency serving pages with slow ad responses
propagations:
  - source: flagd-config
    target: ad-service-1
    condition: Feature flag triggers manual full garbage collection in ad service
    effect: Ad service experiences GC pauses causing increased response times and potential timeouts
  - source: ad-service-1
    target: frontend-service-1
    condition: Frontend service calls to ad service timeout or return slowly due to GC pauses
    effect: Frontend experiences errors and increased latency when loading advertisements
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns slow responses to proxy due to ad service delays
    effect: Users experience slow page loads with delayed or missing advertisements
recommended_actions:
  - solution:
      id: Disable adManualGc flag
      actions:
        - Disable adManualGc flag in flagd configuration
  - solution:
      id: Restart ad service
      actions:
        - Restart ad service to clear any degraded state


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: checkout
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: payment
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: "data.demo\\.flagd\\.json"
        from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: paymentFailure feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: payment-service-1
    kind: Service
    filter:
      - payment\b
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
alerts:
  - id: RequestErrorRate
    group_id: payment-service-1
    metadata:
      description: Payment service charge requests failing due to feature flag
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service experiencing errors when attempting to charge payment
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors during checkout operations
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing errors from failed checkout attempts
propagations:
  - source: flagd-config
    target: payment-service-1
    condition: Feature flag causes payment service charge method to fail
    effect: Payment service returns errors for all charge requests
  - source: payment-service-1
    target: checkout-service-1
    condition: Checkout service cannot successfully charge payment due to payment service failures
    effect: Checkout operations fail when attempting to process payments
  - source: checkout-service-1
    target: frontend-service-1
    condition: Frontend receives errors from checkout when payment processing fails
    effect: Users cannot complete purchases, checkout flow fails
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns error responses to proxy for failed checkout attempts
    effect: Users experience complete checkout failure through the proxy
recommended_actions:
  - solution:
      id: Fix the payment service failure error
      actions:
        - Fix the payment service failure error
  - solution:
      id: Disable paymentFailure flag
      actions:
        - Disable paymentFailure flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: otel-demo-emailservice
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: checkout
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: "data.demo\\.flagd\\.json"
        from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: paymentUnreachable feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: payment-service-1
    kind: Service
    filter:
      - payment\b
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
alerts:
  - id: RequestErrorRate
    group_id: payment-service-1
    metadata:
      description: Payment service unavailable due to feature flag
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service unable to reach payment service
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors during checkout operations
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing errors from failed checkout attempts
propagations:
  - source: flagd-config
    target: payment-service-1
    condition: Feature flag makes payment service completely unavailable
    effect: Payment service cannot be reached by any callers
  - source: payment-service-1
    target: checkout-service-1
    condition: Checkout service cannot connect to payment service at all
    effect: Checkout operations fail when attempting to process payments due to service unavailability
  - source: checkout-service-1
    target: frontend-service-1
    condition: Frontend receives errors from checkout when payment service is unreachable
    effect: Users cannot complete purchases, checkout flow fails
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns error responses to proxy for failed checkout attempts
    effect: Users experience complete checkout failure through the proxy
recommended_actions:
  - solution:
      id: Disable paymentUnreachable flag
      actions:
        - Disable paymentUnreachable flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend-proxy
      kind: Service
    condition: High latency
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: "data.demo\\.flagd\\.json"
        from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: loadgeneratorFloodHomepage feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: load-generator-service-1
    kind: Service
    filter:
      - load-generator\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
alerts:
  - id: RequestLatency
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing high latency due to flood of homepage requests
  - id: RequestLatency
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing high latency from excessive homepage traffic
propagations:
  - source: flagd-config
    target: load-generator-service-1
    condition: Feature flag triggers load generator to flood frontend with high volume of homepage requests
    effect: Load generator creates excessive traffic surge targeting the homepage
  - source: load-generator-service-1
    target: frontend-proxy-service-1
    condition: Frontend proxy receives flood of incoming homepage requests from load generator
    effect: Frontend proxy experiences high latency and resource saturation from excessive traffic
  - source: frontend-proxy-service-1
    target: frontend-service-1
    condition: Frontend proxy forwards high volume of requests to frontend service
    effect: Frontend service experiences high latency processing excessive homepage traffic
recommended_actions:
  - solution:
      id: Identify the home page flood attack source and stop it
      actions:
        - Identify the home page flood attack source and stop it
  - solution:
      id: Disable loadgeneratorFloodHomepage flag
      actions:
        - Disable loadgeneratorFloodHomepage flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: fraud-detection
      kind: Service
    condition: High Kafka consumer lag and slow message processing
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
    changed:
      element: "data.demo\\.flagd\\.json"
      from: "\"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }"
      to: "\"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"on\"\n    }"
    condition: kafkaQueueProblems feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: fraud-detection-service-1
    kind: Service
    filter:
      - fraud-detection\b
    namespace: otel-demo
  - id: kafka-service-1
    kind: Service
    filter:
      - kafka\b
    namespace: otel-demo
alerts:
  - id: RequestLatency
    group_id: fraud-detection-service-1
    metadata:
      description: Fraud detection service experiencing artificial delays from feature flag
  - id: KafkaConnectionClosureRate
    group_id: fraud-detection-service-1
    metadata:
      description: Fraud detection service Kafka consumer experiencing lag and slow message processing
propagations:
  - source: flagd-config
    target: fraud-detection-service-1
    condition: Feature flag causes fraud detection service to sleep 1 second when processing Kafka messages
    effect: Fraud detection consumer processes messages slowly, creating Kafka consumer lag
  - source: fraud-detection-service-1
    target: kafka-service-1
    condition: Slow consumer creates backlog in Kafka queue
    effect: Kafka queue accumulates unprocessed order messages, increasing consumer lag and time between polls
recommended_actions:
  - solution:
      id: Disable kafkaQueueProblems flag
      actions:
        - Disable kafkaQueueProblems flag in flagd configuration


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High HTTP error rate
    category: Resource Saturation
    fault_mechanism: custom(flagd)
  - entity:
      name: product-catalog
      kind: Service
    condition: High HTTP error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      group_id: flagd-config-1
      kind: ConfigMap
    condition: adServiceHighCpu feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
alerts:
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Error Rate is Above Threshold
groups:
  - id: flagd-config-1
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: flagd-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - flagd-.*
  - id: flagd-service-1
    kind: Service
    namespace: otel-demo
    filter:
    - flagd
  - id: recommendation-pod-1
    kind: Pod
    filter:
      - recommendation-.*
    namespace: otel-demo
  - id: recommendation-service-1
    kind: Service
    filter:
    - recommendation\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
    - frontend\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - frontend-.*
  - id: products-endpoint-2
    kind: ServiceOperation
    namespace: otel-demo
    filter:
    - "product-catalog-.*_ProductCatalogService_GetProduct"
  - id: productcatalog-service-2
    kind: Service
    filter:
    - product-catalog\b
    namespace: otel-demo
  - id: checkout-service-2
    kind: Service
    filter:
    - checkout\b
    namespace: otel-demo
  - id: productcatalog-pod-2
    kind: Pod
    namespace: otel-demo
    filter:
      - product-catalog-.*
  - id: checkout-pod-2
    kind: Pod
    namespace: otel-demo
    filter:
      - checkout-.*
aliases:
  - - flagd-pod-1
    - flagd-service-1
  - - frontend-service-1
    - frontend-pod-1
  - - recommendation-service-1
    - recommendation-pod-1
  - - productcatalog-service-2
    - productcatalog-pod-2
    - products-endpoint-2
  - - checkout-service-2
    - checkout-pod-2
propagations:
  - source: flagd-service-1
    target: recommendation-pod-1
    condition: flagd enables injection
    effect: recommendation pod crashLoopBack Failure
  - source: recommendation-pod-1
    target: recommendation-service-1
    condition: recommendation pod crashLoopBack Failure
    effect: recommendation service Error Rate is Above Threshold
  - source: recommendation-service-1
    target: frontend-service-1
    condition: recommendation service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
  - source: flagd-config-1
    target: flagd-service-1
    condition: flagd-config ConfigMap ProductCatalog feature flag set and recommendationCacheFailure feature flag set
    effect: flagd enables injection
  - source: flagd-service-1
    target: products-endpoint-2
    condition: flagd enables injection
    effect: products endpoint GetProduct product id OLJCESPC7Z requests returning errors
  - source: products-endpoint-2
    target: productcatalog-service-2
    condition: products endpoint GetProduct product id OLJCESPC7Z requests returning errors
    effect: productcatalog service Error Rate is Above Threshold
  - source: productcatalog-service-2
    target: checkout-service-2
    condition: productcatalog service Error Rate is Above Threshold
    effect: checkout service Error Rate is Above Threshold
  - source: checkout-service-2
    target: frontend-service-1
    condition: checkout service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
recommended_actions:
  - solution:
      id: Disable_recommendationCacheFailure_feature_flag
      actions:
        - Disable recommendationCacheFailure feature flag
  - solution:
      id: Increase_Memory_Allocation
      actions:
        - Increase Memory Allocation
  - solution:
      id: Disable_ProductCatalogFailure_feature_flag
      actions:
        - Disable ProductCatalogFailure feature flag
  - solution:
      id: Block_traffic_for_product_id_OLJCESPC7Z
      actions:
        - Block traffic for product id OLJCESPC7Z


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: checkout
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: data.demo\\.flagd\\.json
        from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: paymentSeriveFailure and paymentUnreachable feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
groups:
  - id: flagd-config
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: payment-service-1
    kind: Service
    filter:
      - payment\b
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
alerts:
  - id: RequestErrorRate
    group_id: payment-service-1
    metadata:
      description: Payment service unavailable and charge requests failing due to combined feature flags
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service unable to reach payment service and experiencing charge failures
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors during checkout operations
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing errors from failed checkout attempts
propagations:
  - source: flagd-config
    target: payment-service-1
    condition: Combined feature flags make payment service unavailable and cause charge method to fail
    effect: Payment service experiences compound failure with complete unavailability and charge request failures
  - source: payment-service-1
    target: checkout-service-1
    condition: Checkout service cannot connect to payment service and receives charge failures
    effect: Checkout operations fail completely when attempting to process any payments
  - source: checkout-service-1
    target: frontend-service-1
    condition: Frontend receives errors from checkout when payment service is both unreachable and failing
    effect: Users cannot complete purchases, checkout flow completely blocked
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns error responses to proxy for all failed checkout attempts
    effect: Users experience complete checkout failure through the proxy with no ability to complete orders
recommended_actions:
  - solution:
      id: Fix the payment service failure and unreachable error
      actions:
        - Fix the payment service failure and unreachable error
  - solution:
      id: Disable paymentFailure and paymentUnreachable flag
      actions:
        - Disable paymentFailure and paymentUnreachable flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: checkout
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: payment
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: data.demo\\.flagd\\.json
        from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: paymentSeriveFailure and adFailure feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
alerts:
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Error Rate is Above Threshold
  - id: RequestErrorRate
    group_id: checkout-service-2
    metadata:
      description: Error Rate is Above Threshold
    group_id: payment-service-2
  - id: RequestErrorRate
    group_id: checkout-service-2
    metadata:
      description: Error Rate is Above Threshold
groups:
  - id: flagd-config-1
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: flagd-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - flagd-.*
  - id: flagd-service-1
    kind: Service
    namespace: otel-demo
    filter:
    - flagd
  - id: getads-endpoint-1
    kind: ServiceOperation
    filter:
      - ad-.*_Ad_GetAds
    namespace: otel-demo
  - id: ad-service-1
    kind: Service
    filter:
    - ad
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
    - frontend\b
    namespace: otel-demo
  - id: ad-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - ad-.*
  - id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - frontend-.*
  - id: charge-endpoint-2
    kind: ServiceOperation
    namespace: otel-demo
    filter:
    - checkout-.*_PaymentService_Charge
  - id: payment-service-2
    kind: Service
    filter:
    - payment
    namespace: otel-demo
  - id: checkout-service-2
    kind: Service
    filter:
    - .checkout
    namespace: otel-demo
  - id: checkout-pod-2
    kind: Pod
    namespace: otel-demo
    filter:
      - checkout-.*
  - id: payment-pod-2
    kind: Pod
    namespace: otel-demo
    filter:
      - payment-.*
aliases:
  - - flagd-pod-1
    - flagd-service-1
  - - ad-service-1
    - ad-pod-1
    - getads-endpoint-1
  - - frontend-service-1
    - frontend-pod-1
  - - checkout-service-2
    - checkout-pod-2
  - - payment-service-2
    - payment-pod-2
propagations:
  - source: flagd-config-1
    target: flagd-service-1
    condition: flagd-config ConfigMap adFailure feature flag set and paymentFailure feature flag set
    effect: flagd enables injection
  - source: flagd-service-1
    target: getads-endpoint-1
    condition: flagd enables injection
    effect: getads endpoint Error Rate is Above Threshold
  - source: getads-endpoint-1
    target: ad-service-1
    condition: getads endpoint Error Rate is Above Threshold
    effect: ad service Error Rate is Above Threshold
  - source: ad-service-1
    target: frontend-service-1
    condition: ad service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
  - source: flagd-service-1
    target: charge-endpoint-2
    condition: flagd enables injection
    effect: charge endpoint Error Rate is Above Threshold
  - source: charge-endpoint-2
    target: payment-service-2
    condition: charge endpoint Error Rate is Above Threshold
    effect: payment service Error Rate is Above Threshold
  - source: payment-service-2
    target: checkout-service-2
    condition: payment service Error Rate is Above Threshold
    effect: checkout service Error Rate is Above Threshold
  - source: checkout-service-2
    target: frontend-service-1
    condition: checkout service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
recommended_actions:
  - solution:
      id: Disable_adFailure_feature_flag
      actions:
        - Disable adFailure feature flag
  - solution:
      id: Disable_PaymentFailure_feature_flag
      actions:
        - Disable PaymentFailure feature flag


---
metadata:
  version: "v1"
fault:
  - entity:
      name: frontend
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: ad
      kind: Service
    condition: High error rate
    category: Resource Unavailable
    fault_mechanism: custom(flagd)
  - entity:
      name: flagd-config
      kind: ConfigMap
      changed:
        element: data.demo\\.flagd\\.json
        from: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
        to: "{\n  \"$schema\": \"https://flagd.dev/schema/v0/flags.json\",\n  \"flags\": {\n    \"productCatalogFailure\": {\n      \"description\": \"Fail product catalog service on a specific product\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"recommendationCacheFailure\": {\n      \"description\": \"Fail recommendation service cache\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adManualGc\": {\n      \"description\": \"Triggers full manual garbage collections in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adServiceHighCpu\": {\n      \"description\": \"Triggers high cpu load in the ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"adFailure\": {\n      \"description\": \"Fail ad service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"kafkaQueueProblems\": {\n      \"description\": \"Overloads Kafka queue while simultaneously introducing a consumer side delay leading to a lag spike\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"cartFailure\": {\n      \"description\": \"Fail cart service\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentFailure\": {\n      \"description\": \"Fail payment service charge requests\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"paymentUnreachable\": {\n      \"description\": \"Payment service is unavailable\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": true,\n        \"off\": false\n      },\n      \"defaultVariant\": \"on\"\n    },\n    \"loadgeneratorFloodHomepage\": {\n      \"description\": \"Flood the frontend with a large amount of requests.\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"on\": 100,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    },\n    \"imageSlowLoad\": {\n      \"description\": \"slow loading images in the frontend\",\n      \"state\": \"ENABLED\",\n      \"variants\": {\n        \"10sec\": 10000,\n        \"5sec\": 5000,\n        \"off\": 0\n      },\n      \"defaultVariant\": \"off\"\n    }\n  }\n}\n"
    condition: paymentUnreachable and adFailure feature flag set
    category: Configuration Setting
    fault_mechanism: custom(flagd)
alerts:
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Error Rate is Above Threshold
groups:
  - id: flagd-config-1
    kind: ConfigMap
    name: flagd-config
    namespace: otel-demo
    root_cause: true
  - id: flagd-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - flagd-.*
  - id: flagd-service-1
    kind: Service
    namespace: otel-demo
    filter:
    - flagd
  - id: getads-endpoint-1
    namespace: otel-demo
    kind: ServiceOperation
    filter:
      - ad-.*_AdService_GetAds
  - id: ad-service-1
    kind: Service
    filter:
    - ad\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
    - frontend\b
    namespace: otel-demo
  - id: ad-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - ad-.*
  - id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
      - frontend-.*
  - id: payment-service-2
    kind: Service
    filter:
    - payment\b
    namespace: otel-demo
  - id: checkout-service-2
    kind: Service
    filter:
    - checkout\b
    namespace: otel-demo
  - id: checkout-pod-2
    kind: Pod
    namespace: otel-demo
    filter:
      - checkout-.*
  - id: payment-pod-2
    kind: Pod
    namespace: otel-demo
    filter:
      - payment-.*
aliases:
  - - flagd-pod-1
    - flagd-service-1
  - - ad-service-1
    - ad-pod-1
    - getads-endpoint-1
  - - frontend-service-1
    - frontend-pod-1
  - - checkout-service-2
    - checkout-pod-2
  - - payment-service-2
    - payment-pod-2
propagations:
  - source: flagd-config-1
    target: flagd-service-1
    condition: flagd-config ConfigMap adFailure feature flag set and paymentUnreachable feature flag set
    effect: flagd enables injection
  - source: flagd-service-1
    target: getads-endpoint-1
    condition: flagd enables injection
    effect: getads endpoint Error Rate is Above Threshold
  - source: getads-endpoint-1
    target: ad-service-1
    condition: getads endpoint Error Rate is Above Threshold
    effect: ad service Error Rate is Above Threshold
  - source: ad-service-1
    target: frontend-service-1
    condition: ad service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
  - source: flagd-service-1
    target: payment-service-2
    condition: flagd enables injection
    effect: payment service  Unavailable
  - source: payment-service-2
    target: checkout-service-2
    condition: payment service  Unavailable
    effect: checkout service Error Rate is Above Threshold
  - source: checkout-service-2
    target: frontend-service-1
    condition: checkout service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
recommended_actions:
  - solution:
      id: Disable_adFailure_feature_flag
      actions:
        - Disable adFailure feature flag
  - solution:
      id: Disable_paymentUnreachable_feature_flag
      actions:
        - Disable paymentUnreachable feature flag


---
fault:
  - entity:
      name: shipping-deployment-1
      group_id: shipping-deployment-1
      kind: Deployment
    condition: QUOTE_ADDR environment variable set to incorrect port 0000
    category: Change
    fault_mechanism: custom
alerts:
  - id: RequestLatency
    group_id: shipping-service-1
    metadata:
      description: Latency in service shipping in namespace otel-demo is above 1500ms
groups:
  - id: shipping-deployment-1
    kind: Deployment
    filter:
      - shipping\b
    namespace: otel-demo
    root_cause: true
  - id: shipping-service-1
    kind: Service
    filter:
      - shipping\b
    namespace: otel-demo
  - id: shipping-pod-1
    kind: Pod
    filter:
      - shipping-.*
    namespace: otel-demo
aliases:
  - - shipping-service-1
    - shipping-pod-1
    - shipping-deployment-1
propagations:
  - source: shipping-deployment-1
    target: shipping-service-1
    condition: Shipping service cannot connect to quote service due to incorrect QUOTE_ADDR port configuration
    effect: Shipping service experiences high latency while attempting to connect to invalid port
recommended_actions:
  - solution:
      id: fix_quote_port_configuration
      actions:
        - Identify correct port for quote service
        - Update QUOTE_ADDR environment variable in shipping deployment


---
metadata:
  version: "v1"
fault:
  - entity:
      name: delay-otel-demo-product-catalog-1
      group_id: delay-otel-demo-product-catalog-1
      kind: Service
    condition: To Be Specified
    category: Latency
    fault_mechanism: chaos
alerts:
  - id: RequestLatency
    group_id: frontend-service-1
    metadata:
      description: Latency is Above Threshold
  - id: RequestLatency
    group_id: recommendation-service-1
    metadata:
      description: Latency is Above Threshold
  - id: RequestLatency
    group_id: checkout-service-1
    metadata:
      description: Latency is Above Threshold
  - id: RequestLatency
    group_id: productcatalog-service-1
    metadata:
      description: Latency is Above Threshold
  - id: RequestLatency
    group_id: frontendproxy-service-1
    metadata:
      description: Latency is Above Threshold
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Error Rate is Above Threshold
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Error Rate is Above Threshold
  - id: RequestErrorRate
    group_id: recommendation-service-1
    metadata:
      description: Error Rate is Above Threshold
groups:
  - id: delay-otel-demo-product-catalog-1
    kind: NetworkChaos
    filter:
    - .*product-catalog
    namespace: chaos-mesh
    root_cause: true
  - id: productcatalog-service-1
    kind: Service
    filter:
    - product-catalog\b
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
    - checkout\b
    namespace: otel-demo
  - id: recommendation-service-1
    filter:
    - recommendation\b
    kind: Service
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
    - frontend\b
    namespace: otel-demo
  - id: frontendproxy-service-1
    kind: Service
    filter:
    - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
    - frontend-.*
  - id: productcatalog-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
    - product-catalog-.*
  - id: checkout-pod-1
    kind: Pod
    namespace: otel-demo
    filter:
    - checkout-.*
  - id: recommendation-pod-1
    kind: Pod
    filter:
    - recommendation-.*
    namespace: otel-demo
aliases:
  - - frontend-service-1
    - frontend-pod-1
  - - productcatalog-service-1
    - productcatalog-pod-1
  - - checkout-service-1
    - checkout-pod-1
  - - recommendation-service-1
    - recommendation-pod-1
propagations:
  - source: delay-otel-demo-product-catalog-1
    target: productcatalog-service-1
    condition: delay-otel-demo-productcatalogservice networkchaos productcatalogservice
      network delay set to 20 seconds
    effect: productcatalog service Error Rate is Above Threshold
  - source: productcatalog-service-1
    target: checkout-service-1
    condition: Latency is too high or Error Rate is Above Threshold
    effect: checkout service Error Rate is Above Threshold
  - source: productcatalog-service-1
    target: recommendation-service-1
    condition: Latency is too high or Error Rate is Above Threshold
    effect: recommendation service Error Rate is Above Threshold
  - source: productcatalog-service-1
    target: frontend-service-1
    condition: Latency is too high or Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
recommended_actions:
  - solution:
      id: Delete_the_resource_networkchaos_productcatalogservice
      actions:
        - Delete the resource networkchaos productcatalogservice


---
metadata:
  version: "v1"
alerts:
  - group_id: frontend-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
aliases:
  - - checkout-service-1
    - checkout-pod-1
  - - frontend-service-1
    - frontend-pod-1
  - - otel-demo-product-catalog-network-delay
    - otel-demo-product-catalog-network-delay-1
fault:
  - category: Resource Unavailable
    condition: To Be Specified
    entity:
      group_id: pod-otel-demo-checkout-1
      kind: Service
      name: pod-otel-demo-checkout-1
    fault_mechanism: chaos
groups:
  - filter:
      - .*product-catalog-network-delay
    id: otel-demo-product-catalog-network-delay-1
    kind: PodChaos
    namespace: chaos-mesh
    root_cause: true
  - filter:
      - .*product-catalog-network-delay
    id: otel-demo-product-catalog-network-delay
    kind: Schedule
    namespace: chaos-mesh
  - filter:
      - .*product-catalog-network-delay
    id: otel-demo-product-catalog-network-delay
    kind: Schedule
    namespace: chaos-mesh
  - filter:
      - checkout-.*
    id: checkout-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - frontend\b
    id: frontend-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - checkout\b
    id: checkout-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - frontend-.*
    id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
propagations:
  - condition: set pod-failure on checkoutservice
    effect: checkoutservice pod is unavailable
    source: otel-demo-product-catalog-network-delay-1
    target: checkout-pod-1
  - condition: checkoutservice pod is unavailable
    effect: frontend service Error Rate is Above Threshold
    source: checkout-pod-1
    target: frontend-service-1
recommended_actions:
  - solution:
      actions:
        - Delete the resource podchaos checkoutservice
      id: Delete_the_resource_podchaos_checkoutservice
  - solution:
      actions:
        - restart the checkoutservice pod
      id: restart_the_checkoutservice_pod