---
metadata:
  version: "v1"
alerts:
  - group_id: frontend-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
  - group_id: shipping-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
  - group_id: checkout-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
  - group_id: frontend-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
  - group_id: checkout-service-1
    id: RequestLatency
    metadata:
      description: Latency is Above Threshold
  - group_id: shipping-service-1
    id: RequestLatency
    metadata:
      description: Latency is Above Threshold
aliases:
  - - frontend-service-1
    - frontend-pod-1
  - - checkout-service-1
    - checkout-pod-1
  - - shipping-service-1
    - shipping-pod-1
  - - quote-service-1
    - quote-pod-1
  - - otel-demo-email-checkout-network-partition-1
    - otel-demo-email-checkout-network-partition
fault:
  - category: Resource Unavailable
    condition: To Be Specified
    entity:
      group_id: partition-otel-demo-shippingservice-1
      kind: Service
      name: partition-otel-demo-shippingservice-1
    fault_mechanism: chaos
groups:
  - filter:
      - .*email-checkout-network-partition
    id: otel-demo-email-checkout-network-partition-1
    kind: NetworkChaos
    namespace: chaos-mesh
    root_cause: true
  - filter:
      - .*email-checkout-network-partition
    id: otel-demo-email-checkout-network-partition
    kind: Schedule
    namespace: chaos-mesh
  - filter:
      - shipping\b
    id: shipping-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - checkout\b
    id: checkout-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - frontend\b
    id: frontend-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - frontend-.*
    id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - checkout-.*
    id: checkout-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - shipping-.*
    id: shipping-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - quote-.*
    id: quote-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - quote\b
    id: quote-service-1
    kind: Service
    namespace: otel-demo
propagations:
  - condition: email-checkout-network-partition networkchaos partitioning network from shippingservice to quoteservice
    effect: quote service Error Rate is Above Threshold
    source: otel-demo-email-checkout-network-partition-1
    target: quote-service-1
  - condition: Latency is too high or quote service Error Rate is Above Threshold
    effect: shipping service Error Rate is Above Threshold
    source: quote-service-1
    target: shipping-service-1
  - condition: Latency is too high or shipping service Error Rate is Above Threshold
    effect: checkout service Error Rate is Above Threshold
    source: shipping-service-1
    target: checkout-service-1
  - condition: checkout service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
    source: checkout-service-1
    target: frontend-service-1
recommended_actions:
  - solution:
      actions:
        - Delete the resource networkchaos shippingservice
      id: Delete_the_resource_networkchaos_shippingservice


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-20
spec:
  alerts:
    - group_id: frontend-service-1
      id: RequestErrorRate
      metadata:
        description: Error Rate is Above Threshold
    - group_id: recommendation-service-1
      id: RequestErrorRate
      metadata:
        description: Error Rate is Above Threshold
  aliases:
    - - productcatalog-service-1
      - productcatalog-pod-1
    - - productcatalog-pod-1
      - productcatalog-deployment-1
    - - frontend-service-1
      - frontend-pod-1
    - - checkout-service-1
      - checkout-pod-1
    - - recommendation-service-1
      - recommendation-pod-1
  fault:
    - category: Change
      condition: To Be Specified
      entity:
        group_id: checkout-pod-1
        kind: Pod
        name: productcatalog-pod-1
      fault_mechanism: custom
  groups:
    - filter:
        - product-catalog-.*
      id: productcatalog-deployment-1
      kind: Deployment
      namespace: otel-demo
      root_cause: true
    - filter:
        - product-catalog-.*
      id: productcatalog-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - checkout\b
      id: checkout-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - recommendation\b
      id: recommendation-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend\b
      id: frontend-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - product-catalog\b
      id: productcatalog-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-.*
      id: frontend-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - checkout-.*
      id: checkout-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - recommendation-.*
      id: recommendation-pod-1
      kind: Pod
      namespace: otel-demo
  propagations:
    - condition: productcatalogdeployement updated to point to an invalid image - incorrect image path-location
      effect: productcatalog pod is in ImagePullBackOff error
      source: productcatalog-deployment-1
      target: productcatalog-pod-1
    - condition: productcatalog pod is in ImagePullBackOff error
      effect: frontend service Error Rate is Above Threshold
      source: productcatalog-pod-1
      target: frontend-service-1
    - condition: productcatalog pod is in ImagePullBackOff error
      effect: checkout service Error Rate is Above Threshold
      source: productcatalog-pod-1
      target: checkout-service-1
    - condition: checkout service Error Rate is Above Threshold
      effect: frontend service Error Rate is Above Threshold
      source: checkout-service-1
      target: frontend-service-1
    - condition: productcatalog pod is in ImagePullBackOff error
      effect: recommendation service Error Rate is Above Threshold
      source: productcatalog-pod-1
      target: recommendation-service-1
    - condition: recommendation service Error Rate is Above Threshold
      effect: frontend service Error Rate is Above Threshold
      source: recommendation-service-1
      target: frontend-service-1
  recommended_actions:
    - solution:
        actions:
          - change the image field in productcatalogueservice deployment pointing to the correct image
        id: change_the_image_field_in_productcatalogueservice_deployment_pointing_to_the_correct_image


---
metadata:
  version: "v1"
fault:
  - entity:
      name: otel-demo-valkey-memory-stress
      group_id: otel-demo-valkey-memory-stress
      kind: StressChaos
    condition: Chaos Mesh stress injecting memory pressure on valkey-cart pods
    category: Resource
    fault_mechanism: memory_stress
alerts:
  - id: FailedPodsDetected
    group_id: valkey-cart-pod-1
    metadata:
      description: Valkey cart pods being terminated or restarted due to OOM conditions
  - id: RequestErrorRate
    group_id: cart-service-1
    metadata:
      description: Cart service experiencing errors when accessing valkey cache
  - id: RequestLatency
    group_id: cart-service-1
    metadata:
      description: Cart service experiencing increased latency due to cache unavailability
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors when accessing cart functionality
  - id: RequestLatency
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing increased latency for cart operations
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service experiencing errors when retrieving cart data
  - id: RequestLatency
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing increased latency serving pages with cart operations
groups:
  - id: otel-demo-valkey-memory-stress
    kind: StressChaos
    filter:
      - otel-demo-valkey-memory-stress
    namespace: chaos-mesh
    root_cause: true
  - id: valkey-cart-pod-1
    kind: Pod
    filter:
      - valkey-cart-.*
    namespace: otel-demo
  - id: valkey-cart-service-1
    kind: Service
    filter:
      - valkey-cart\b
    namespace: otel-demo
  - id: cart-service-1
    kind: Service
    filter:
      - cart\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
aliases:
  - - valkey-cart-service-1
    - valkey-cart-pod-1
propagations:
  - source: otel-demo-valkey-memory-stress
    target: valkey-cart-pod-1
    condition: Chaos Mesh StressChaos resource injects memory stress on valkey-cart pods
    effect: Valkey cart pods experience high memory usage and potential OOM kills
  - source: valkey-cart-pod-1
    target: valkey-cart-service-1
    condition: Valkey pods under memory pressure respond slowly, fail requests, or get terminated
    effect: Valkey cart cache becomes unavailable or severely degraded
  - source: valkey-cart-service-1
    target: cart-service-1
    condition: Cart service cannot access valkey cache for storing and retrieving cart data
    effect: Cart operations fail, users cannot add items to cart or view cart contents
  - source: cart-service-1
    target: frontend-service-1
    condition: Frontend receives errors from cart service when users interact with cart
    effect: Cart functionality fails for users browsing and shopping
  - source: cart-service-1
    target: checkout-service-1
    condition: Checkout service cannot retrieve cart data from cart service
    effect: Checkout operations fail as cart contents are unavailable
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns error responses to proxy for failed cart and checkout operations
    effect: Users cannot use cart functionality or complete purchases through the proxy
recommended_actions:
  - solution:
      id: resolve_memory_stress
      actions:
        - Identify and stop the stress chaos experiment affecting valkey-cart service
        - Monitor valkey memory usage and pod health


---
metadata:
  version: "v1"
fault:
  - entity:
      name: otel-demo-ad-memory-stress
      group_id: otel-demo-ad-memory-stress
      kind: StressChaos
    condition: Chaos Mesh stress injecting memory pressure on ad service pods
    category: Resource
    fault_mechanism: memory_stress
alerts:
  - id: RequestLatency
    group_id: ad-service-1
    metadata:
      description: Ad service experiencing increased response times due to memory pressure
  - id: FailedPodsDetected
    group_id: ad-pod-1
    metadata:
      description: Ad pods being terminated or restarted due to OOM conditions
  - id: RequestLatency
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing increased latency waiting for ad service responses
  - id: RequestLatency
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing increased latency serving pages with slow or failed ad responses
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors when calling ad service
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing increased error rate
groups:
  - id: otel-demo-ad-memory-stress
    kind: StressChaos
    filter:
      - otel-demo-ad-memory-stress
    namespace: chaos-mesh
    root_cause: true
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
  - id: ad-service-1
    kind: Service
    filter:
      - ad\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
aliases:
  - - ad-service-1
    - ad-pod-1
propagations:
  - source: otel-demo-ad-memory-stress
    target: ad-pod-1
    condition: Chaos Mesh StressChaos resource injects memory stress on ad service pods
    effect: Ad pods experience high memory usage and potential OOM kills
  - source: ad-pod-1
    target: ad-service-1
    condition: Ad pods under memory pressure respond slowly, fail requests, or get terminated
    effect: Ad service experiences degraded performance with high latency, errors, or complete unavailability
  - source: ad-service-1
    target: frontend-service-1
    condition: Frontend service calls to ad service fail or return slowly
    effect: Frontend pages load slowly or without advertisements, degrading user experience
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend service returns slow or error responses to proxy due to ad service issues
    effect: Users experience slow page loads or pages without ads through the proxy
recommended_actions:
  - solution:
      id: resolve_memory_stress
      actions:
        - Identify and stop the stress chaos experiment affecting ad service


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-23
spec:
  alerts:
    - group_id: checkout-pod-1
      id: PendingPodsDetected
      metadata:
        description: Pods in pending state for checkout service
  aliases:
    - - checkout-service-1
      - checkout-pod-1
  fault:
    - category: Change
      condition: To Be Specified
      entity:
        group_id: checkout-pod-1
        kind: Pod
        name: checkout-pod-1
      fault_mechanism: custom
  groups:
    - filter:
        - checkout-.*
      id: checkout-deployment-1
      kind: Deployment
      namespace: otel-demo
      root_cause: true
    - filter:
        - checkout-.*
      id: checkout-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - checkout\b
      id: checkout-service-1
      kind: Service
      namespace: otel-demo
  propagations:
    - condition: checkout deployment updated to point to an invalid image - incorrect architecture
      effect: checkout pod is in ImagePullBackOff error
      source: checkout-deployment-1
      target: checkout-pod-1
    - condition: checkout pod is in ImagePullBackOff error
      effect: ''
      source: checkout-pod-1
      target: checkout-service-1
  recommended_actions:
    - solution:
        actions:
          - change the image of checkout to correct image for amd64 system
        id: change_the_image_of_checkout_to_correct_image_for_amd64_system


---
metadata:
  version: "v1"
fault:
  - entity:
      name: checkout-deployment
      group_id: checkout-deployment
      kind: Deployment
    condition: Checkout deployment has misconfigured KAFKA_ADDR environment variable pointing to wrong port 9999
    category: Configuration
    fault_mechanism: environment_variable_misconfiguration
alerts:
  - id: KafkaConnectionClosureRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service unable to connect to Kafka due to misconfigured port
  - id: RequestLatency
    group_id: checkout-service-1
    metadata:
      description: Checkout service experiencing increased latency due to Kafka connection failures
groups:
  - id: checkout-deployment
    kind: Deployment
    filter:
      - checkout\b
    namespace: otel-demo
    root_cause: true
  - id: checkout-pod-1
    kind: Pod
    filter:
      - checkout-.*
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: kafka-service-1
    kind: Service
    filter:
      - kafka\b
    namespace: otel-demo
aliases:
  - - checkout-service-1
    - checkout-pod-1
propagations:
  - source: checkout-deployment
    target: checkout-pod-1
    condition: Deployment configuration specifies incorrect Kafka port 9999 instead of correct port
    effect: Checkout pods cannot establish connections to Kafka on the wrong port
  - source: checkout-pod-1
    target: checkout-service-1
    condition: Checkout pods fail to connect to Kafka due to misconfigured environment variable
    effect: Checkout service cannot publish order events to Kafka, but continues processing orders
  - source: checkout-service-1
    target: kafka-service-1
    condition: Checkout service attempts to connect to Kafka on wrong port 9999
    effect: Order events are not published to Kafka queue, downstream services miss order notifications
recommended_actions:
  - solution:
      id: fix_kafka_environment_variable
      actions:
        - Correct KAFKA_ADDR environment variable in checkout deployment to use the correct Kafka port


---
metadata:
  version: "v1"
alerts:
  - group_id: frontend-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
  - group_id: recommendation-service-1
    id: RequestErrorRate
    metadata:
      description: Error Rate is Above Threshold
aliases:
  - - frontend-service-1
    - frontend-pod-1
  - - recommendation-pod-1
    - recommendation-service-1
fault:
  - category: Other
    condition: Resource Saturation
    entity:
      group_id: cpustress-recommendation-1
      kind: Service
      name: cpustress-recommendation-1
    fault_mechanism: chaos
groups:
  - filter:
      - .*recommendation
    id: otel-demo-recommendation-cpu-stress
    kind: Schedule
    namespace: chaos-mesh
    root_cause: true
  - filter:
      - recommendation-.*
    id: recommendation-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - recommendation\b
    id: recommendation-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - frontend\b
    id: frontend-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - frontend-.*
    id: frontend-pod-1
    kind: Pod
    namespace: otel-demo
propagations:
  - condition: chaosmesh cpustresser hogs cpu
    effect: recommendationservice pod cpu saturation
    source: otel-demo-recommendation-cpu-stress
    target: recommendation-pod-1
  - condition: recommendationservice cpu saturation
    effect: recommendationservice Error Rate is Above Threshold. frontend service Error Rate is Above Threshold
    source: recommendation-pod-1
    target: frontend-service-1
recommended_actions:
  - solution:
      actions:
        - Delete the resource stresschaos recommendationservice
      id: Delete_the_resource_stresschaos_recommendationservice
  - solution:
      actions:
        - Increase CPU Allocation
      id: Increase_CPU_Allocation


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-26
spec:
  alerts:
    - group_id: checkout-service-1
      id: HighRequestErrorRate
      metadata:
        description: Request error rate in service checkout in namespace otel-demo is above 0
    - group_id: checkout-service-1
      id: HighRequestLatency
      metadata:
        description: Checkout service latency increases when email service fails
    - group_id: frontend-service-1
      id: HighRequestErrorRate
      metadata:
        description: Frontend errors cascade from checkout service failures
    - group_id: frontend-proxy-service-1
      id: HighRequestErrorRate
      metadata:
        description: Frontend-proxy errors cascade from checkout service failures
  aliases:
    - - email-service-1
      - email-pod-1
    - - checkout-service-1
      - checkout-pod-1
  fault:
    - category: Chaos
      condition: HTTP Chaos fault tampering POST request body to email service with corrupted JSON payload
      entity:
        group_id: tamper-body-otel-demo-emailservice-1
        kind: HttpChaos
        name: tamper-body-otel-demo-emailservice-1
      fault_mechanism: chaos
  groups:
    - filter:
        - otel-demo-email-http-chaos-post-body-tamper
      id: tamper-body-otel-demo-emailservice-1
      kind: HttpChaos
      namespace: chaos-mesh
      root_cause: true
    - filter:
        - email\b
      id: email-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - email-.*
      id: email-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - checkout\b
      id: checkout-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - checkout-.*
      id: checkout-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend\b
      id: frontend-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-proxy
      id: frontend-proxy-service-1
      kind: Service
      namespace: otel-demo
  propagations:
    - condition: HTTP Chaos tampers POST request bodies with invalid JSON payload
      effect: Email service receives corrupted requests and fails to process them
      source: tamper-body-otel-demo-emailservice-1
      target: email-service-1
    - condition: Email service returns errors due to corrupted request payload
      effect: Checkout service experiences errors when sending order confirmation emails
      source: email-service-1
      target: checkout-service-1
    - condition: Email service errors cause retry logic in checkout
      effect: Checkout service experiences increased latency from email service retries
      source: email-service-1
      target: checkout-service-1
    - condition: Checkout service errors from email failures
      effect: Frontend experiences errors when calling checkout
      source: checkout-service-1
      target: frontend-service-1
    - condition: Frontend errors from checkout failures
      effect: Frontend-proxy experiences cascading errors
      source: frontend-service-1
      target: frontend-proxy-service-1
  recommendedActions:
    - solution:
        actions:
          - Delete the HTTP Chaos resource tampering email service requests
        id: remove_chaos_experiment


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-27
spec:
  alerts:
    - group_id: frontend-proxy-service-1
      id: HighRequestErrorRate
      metadata:
        description: Error rate is above threshold for frontend-proxy service
    - group_id: frontend-service-1
      id: HighRequestErrorRate
      metadata:
        description: Error rate is above threshold for frontend service
    - group_id: checkout-service-1
      id: HighRequestErrorRate
      metadata:
        description: Error rate is above threshold for checkout service
    - group_id: shipping-service-1
      id: HighRequestErrorRate
      metadata:
        description: Error rate is above threshold for shipping service
    - group_id: accounting-service-1
      id: HighRequestLatency
      metadata:
        description: Accounting experiences latency during order processing issues
  aliases:
    - - quote-service-1
      - quote-pod-1
    - - checkout-service-1
      - checkout-pod-1
    - - shipping-service-1
      - shipping-pod-1
    - - frontend-service-1
      - frontend-pod-1
    - - frontend-proxy-service-1
      - frontend-proxy-pod-1
    - - accounting-service-1
  fault:
    - category: Chaos
      condition: HTTP Chaos fault aborting POST requests to quote service /getquote endpoint
      entity:
        group_id: http-abort-otel-demo-quoteservice-1
        kind: HttpChaos
        name: http-abort-otel-demo-quoteservice-1
      fault_mechanism: chaos
  groups:
    - filter:
        - otel-demo-quote-http-chaos-abort
      id: otel-demo-quote-http-chaos-abort
      kind: Schedule
      namespace: chaos-mesh
      root_cause: true
    - filter:
        - quote\b
      id: quote-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - quote-.*
      id: quote-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - checkout\b
      id: checkout-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - checkout-.*
      id: checkout-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - shipping\b
      id: shipping-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - shipping-.*
      id: shipping-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend\b
      id: frontend-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-.*
      id: frontend-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend-proxy\b
      id: frontend-proxy-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-proxy-.*
      id: frontend-proxy-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - accounting\b
      id: accounting-service-1
      kind: Service
      namespace: otel-demo
  propagations:
    - condition: HTTP Chaos aborts all POST requests to quote service /getquote endpoint
      effect: Quote service becomes unavailable for shipping quote requests
      source: otel-demo-quote-http-chaos-abort
      target: quote-service-1
    - condition: Shipping service cannot get quotes due to quote service failures
      effect: Shipping service experiences errors when calling quote service
      source: quote-service-1
      target: shipping-service-1
    - condition: Checkout service receives errors from shipping service due to quote failures
      effect: Checkout service experiences errors during shipping calculations
      source: shipping-service-1
      target: checkout-service-1
    - condition: Frontend receives errors from checkout service due to shipping/quote failures
      effect: Frontend service experiences errors during checkout process
      source: checkout-service-1
      target: frontend-service-1
    - condition: Frontend-proxy receives errors from frontend service due to checkout failures
      effect: Frontend-proxy service experiences errors when proxying requests
      source: frontend-service-1
      target: frontend-proxy-service-1
    - condition: Checkout failures cause order processing delays
      effect: Accounting experiences latency processing incomplete order events
      source: checkout-service-1
      target: accounting-service-1
  recommended_actions:
    - solution:
        actions:
          - Delete the HTTP Chaos resource targeting quote service
        id: remove_chaos_experiment


---
metadata:
  version: "v1"
fault:
  - entity:
      name: jvm-return-otel-demo-ad-1
      group_id: jvm-return-otel-demo-ad-1
      kind: Service
    condition: To Be Specified
    category: Change
    fault_mechanism: chaos
alerts:
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Error rate is above threshold for frontend service
  - id: RequestErrorRate
    group_id: ad-service-1
    metadata:
      description: Error rate is above threshold for ad service
groups:
  - id: jvm-return-otel-demo-ad-1
    kind: JVMChaos
    filter:
      - .*adservice
    namespace: chaos-mesh
    root_cause: true
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
  - id: ad-service-1
    kind: Service
    filter:
      - ad\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-pod
    kind: Pod
    namespace: otel-demo
    filter:
      - frontend-.*
aliases:
  - - frontend-service-1
    - frontend-pod
  - - ad-service-1
    - ad-pod-1
propagations:
  - source: jvm-return-otel-demo-ad-1
    target: ad-pod-1
    condition: Return null for method on adservice
    effect: ad pod null service object
  - source: ad-pod-1
    target: ad-service-1
    condition: ad pod null service object
    effect: ad service Error Rate is Above Threshold
  - source: ad-service-1
    target: frontend-service-1
    condition: ad service Error Rate is Above Threshold
    effect: frontend service Error Rate is Above Threshold
recommended_actions:
  - solution:
      id: Delete_the_resource_jvm-return_on_adservice
      actions:
        - Delete the resource jvm-return on adservice


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-31
spec:
  alerts:
    - group_id: frontend-proxy-service-1
      id: RequestErrorRate
      metadata:
        description: Frontend proxy cannot reach frontend service due to network policy blocking ingress
  aliases:
    - - frontend-service-1
      - frontend-pod-1
    - - frontend-proxy-service-1
      - frontend-proxy-pod-1
  fault:
    - category: Configuration
      condition: NetworkPolicy blocks all ingress traffic to frontend pods
      entity:
        group_id: frontend-pod-1
        kind: Pod
        name: frontend-pod-1
      fault_mechanism: misconfigured-network-policy
  groups:
    - filter:
        - frontend-block-all-ports
      id: frontend-block-all-ports
      kind: NetworkPolicy
      namespace: otel-demo
      root_cause: true
    - filter:
        - frontend-.*
      id: frontend-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend\b
      id: frontend-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-proxy-.*
      id: frontend-proxy-pod-1
      kind: Pod
    - filter:
        - frontend-proxy\b
      id: frontend-proxy-service-1
      kind: Service
      namespace: otel-demo
  propagations:
    - condition: Misconfigured network policy
      effect: NetworkPolicy blocks all ingress traffic to frontend pods
      source: frontend-block-all-ports
      target: frontend-pod-1
    - condition: NetworkPolicy blocks all ingress traffic to frontend pods
      effect: Frontend service cannot receive traffic, making it completely unavailable
      source: frontend-pod-1
      target: frontend-service-1
    - condition: Frontend proxy cannot connect to frontend service due to network policy
      effect: All user requests through proxy fail with connection errors
      source: frontend-service-1
      target: frontend-proxy-1
  recommended_actions:
    - solution:
        actions:
          - Identify and remove the NetworkPolicy blocking ingress to frontend pods
        id: remove_blocking_network_policy


---
metadata:
  version: "v1"
fault:
  - entity:
      name: ad-pod-1
      group_id: ad-pod-1
      kind: Pod
    condition: Ad deployment pods cannot be scheduled due to invalid node selector
    category: Configuration
    fault_mechanism: invalid-node-selector
alerts:
  - id: PendingPodsDetected
    group_id: ad-pod-1
    metadata:
      description: Ad pods stuck in Pending state due to invalid node selector
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing connection failures to ad service
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing errors when serving pages that require ads functionality
groups:
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
    root_cause: true
  - id: ad-service-1
    kind: Service
    filter:
      - ad\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    filter:
      - frontend-.*
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-pod-1
    kind: Pod
    filter:
      - frontend-proxy-.*
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
aliases:
  - - ad-service-1
    - ad-pod-1
  - - frontend-service-1
    - frontend-pod-1
  - - frontend-proxy-service-1
    - frontend-proxy-pod-1
propagations:
  - source: ad-pod-1
    target: ad-service-1
    condition: Ad pods cannot be scheduled, no backend pods available for service
    effect: Ad service has no healthy endpoints, becomes completely unavailable
  - source: ad-service-1
    target: frontend-service-1
    condition: Frontend service cannot connect to ad service with no available endpoints
    effect: Frontend fails to retrieve ads and displays pages with missing ad content
  - source: frontend-service-1
    target: frontend-proxy-1
    condition: Frontend service returns degraded responses with missing ads errors
    effect: Frontend service has slower and incorrect responses
recommended_actions:
  - solution:
      id: fix_node_selectors
      actions:
        - Review and correct node selector configuration for ad and cart deployments


---
fault:
  - entity:
      name: valkey-pod-1
      group_id: valkey-pod-1
      kind: Pod
    condition: Password authentication enabled on valkey-cart service without updating cart service configuration
    category: Configuration
    fault_mechanism: custom
alerts:
  - id: RequestLatency
    group_id: cart-service-1
    metadata:
      description: Latency in service cart in namespace otel-demo is above 1500ms
  - id: RequestErrorRate
    group_id: cart-service-1
    metadata:
      description: Request error rate in service cart in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Request error rate in service checkout in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Request error rate in service frontend in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Request error rate in service frontend-proxy in namespace otel-demo is above 0
groups:
  - id: valkey-pod-1
    kind: Pod
    filter:
      - valkey-cart-.*
    namespace: otel-demo
    root_cause: true
  - id: valkey-service-1
    kind: Service
    filter:
      - valkey-cart\b
    namespace: otel-demo
  - id: cart-service-1
    kind: Service
    filter:
      - cart\b
    namespace: otel-demo
  - id: cart-pod-1
    kind: Pod
    filter:
      - cart-.*
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: checkout-pod-1
    kind: Pod
    filter:
      - checkout-.*
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    filter:
      - frontend-.*
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-proxy-pod-1
    kind: Pod
    filter:
      - frontend-proxy-.*
    namespace: otel-demo
aliases:
  - - valkey-service-1
    - valkey-pod-1
  - - cart-service-1
    - cart-pod-1
  - - checkout-service-1
    - checkout-pod-1
  - - frontend-service-1
    - frontend-pod-1
  - - frontend-proxy-service-1
    - frontend-proxy-pod-1
propagations:
  - source: valkey-pod-1
    target: cart-service-1
    condition: Cart service cannot authenticate to valkey-cart due to password mismatch
    effect: Cart service experiences high latency and errors when accessing cart data
  - source: cart-service-1
    target: checkout-service-1
    condition: Checkout service receives errors when trying to retrieve cart information
    effect: Checkout service experiences errors during cart operations
  - source: checkout-service-1
    target: frontend-service-1
    condition: Frontend receives errors from checkout service due to cart failures
    effect: Frontend service experiences errors during cart and checkout operations
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend-proxy receives errors from frontend service due to cart/checkout failures
    effect: Frontend-proxy service experiences errors when proxying requests
recommended_actions:
  - solution:
      id: fix_cart_authentication
      actions:
        - Update cart service configuration with correct valkey-cart password
        - Restart cart service pods to apply new authentication settings
        - Verify cart service can successfully connect to valkey-cart


---
metadata:
  version: "v1"
fault:
  - entity:
      name: otel-demo-ad-jvm-chaos
      group_id: otel-demo-ad-jvm-chaos
      kind: JVMChaos
    condition: Chaos Mesh JVM stress injecting heap memory pressure on ad service
    category: Resource
    fault_mechanism: jvm_heap_stress
alerts:
  - id: RequestLatency
    group_id: ad-service-1
    metadata:
      description: Ad service experiencing increased response times due to memory pressure
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing timeouts or errors when calling ad service
  - id: RequestLatency
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing increased latency waiting for ad service responses
  - id: RequestLatency
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing increased latency serving pages with slow ad responses
groups:
  - id: otel-demo-ad-jvm-chaos
    kind: JVMChaos
    filter:
      - otel-demo-ad-jvm-chaos
    namespace: chaos-mesh
    root_cause: true
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
  - id: ad-service-1
    kind: Service
    filter:
      - ad\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
aliases:
  - - ad-service-1
    - ad-pod-1
propagations:
  - source: otel-demo-ad-jvm-chaos
    target: ad-pod-1
    condition: Chaos Mesh JVMChaos resource injects heap memory stress on ad service JVM
    effect: Ad pods experience high heap memory usage and frequent garbage collection
  - source: ad-pod-1
    target: ad-service-1
    condition: Ad pods under memory pressure respond slowly or fail requests
    effect: Ad service experiences degraded performance with high latency and potential timeouts
  - source: ad-service-1
    target: frontend-service-1
    condition: Frontend service calls to ad service timeout or return slowly
    effect: Frontend pages load slowly or without advertisements, degrading user experience
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend service returns slow responses to proxy due to ad service delays
    effect: Users experience slow page loads through the proxy
recommended_actions:
  - solution:
      id: resolve_jvm_chaos
      actions:
        - Identify and stop the JVM chaos experiment affecting ad service
        - Monitor ad service heap memory usage and garbage collection metrics


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-37
spec:
  alerts:
    - group_id: otel-demo-namespace-1
      id: CPUSpend
      metadata:
        description: CPU spend increased by 20 percent
    - group_id: otel-demo-namespace-1
      id: MemorySpend
      metadata:
        description: Memory spend has increased by 20 percent
  aliases:
    - - otel-demo-namespace-1
  fault: []
  groups:
    - id: otel-demo-namespace-1
      kind: Namespace
      name: otel-demo
      namespace: otel-demo
      root_cause: true
  propagations: []
  recommendedActions:
    - solution:
        actions:
          - no changes is needed in application
          - update opencost alert to prevent false alerts
        id: no_action


---
fault:
  - entity:
      name: otel-demo-*
      kind: HorizontalPodAutoscaler
      group_id: otel-demo-horizontal-pod-autoscalers
    condition: Misconfigured HPA
    category: Other
    fault_mechanism: custom
alerts:
  - id: high_cost_variation
    text: Cost has increased significantly despite having low utilization
    group_id: otel-demo-namespace-1
groups:
  - id: otel-demo-namespace-1
    kind: Namespace
    name: otel-demo
    namespace:
  - id: otel-demo-horizontal-pod-autoscalers
    kind: HorizontalPodAutoscaler
    namespace: otel-demo
    filter:
      - "*.*"
    root_cause: true
recommended_actions:
  - solution:
      id: investigate_detect_misconfigured_autoscaler
      actions:
        - check current and recent past utilization numbers through promethous
        - found a temporary spike in application load
        - detect misconfiguration in auto-scaler for scaling down


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-40
spec:
  alerts:
    - group_id: valkey-pod-1
      id: KubePodCrashLooping
      metadata:
        description: Valkey-cart pod crashing due to storage exhaustion
    - group_id: cart-service-1
      id: HighRequestErrorRate
      metadata:
        description: Request error rate in service cart in namespace otel-demo is above 0
    - group_id: cart-service-1
      id: HighRequestLatency
      metadata:
        description: Latency in service cart in namespace otel-demo is above 1500ms
    - group_id: checkout-service-1
      id: HighRequestErrorRate
      metadata:
        description: Request error rate in service checkout in namespace otel-demo is above 0
    - group_id: checkout-service-1
      id: HighRequestLatency
      metadata:
        description: Latency in service checkout in namespace otel-demo propagated from cart service delays
    - group_id: frontend-service-1
      id: HighRequestErrorRate
      metadata:
        description: Request error rate in service frontend in namespace otel-demo is above 0
    - group_id: frontend-service-1
      id: HighRequestLatency
      metadata:
        description: Latency in service frontend in namespace otel-demo propagated from checkout service delays
    - group_id: frontend-proxy-service-1
      id: HighRequestErrorRate
      metadata:
        description: Request error rate in service frontend-proxy in namespace otel-demo is above 0
    - group_id: frontend-proxy-service-1
      id: HighRequestLatency
      metadata:
        description: Latency in service frontend-proxy in namespace otel-demo propagated from frontend service delays
  aliases:
    - - valkey-service-1
      - valkey-pod-1
    - - cart-service-1
      - cart-pod-1
    - - checkout-service-1
      - checkout-pod-1
    - - frontend-service-1
      - frontend-pod-1
    - - frontend-proxy-service-1
      - frontend-proxy-pod-1
  fault:
    - category: Resource
      condition: Storage filled to 90% capacity by sidecar container writing dummy data
      entity:
        group_id: valkey-pod-1
        kind: Pod
        name: valkey-cart-pod
      fault_mechanism: custom
  groups:
    - filter:
        - valkey-cart-.*
      id: valkey-pod-1
      kind: Pod
      namespace: otel-demo
      root_cause: true
    - filter:
        - valkey-cart\b
      id: valkey-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - cart\b
      id: cart-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - cart-.*
      id: cart-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - checkout\b
      id: checkout-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - checkout-.*
      id: checkout-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend\b
      id: frontend-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-.*
      id: frontend-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - frontend-proxy\b
      id: frontend-proxy-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - frontend-proxy-.*
      id: frontend-proxy-pod-1
      kind: Pod
      namespace: otel-demo
  propagations:
    - condition: Storage filled to 90% capacity by sidecar container
      effect: Valkey pod crashes repeatedly due to storage exhaustion
      source: valkey-pod-1
      target: valkey-pod-1
    - condition: Valkey storage full, unable to write new cart data
      effect: Cart service experiences write failures and errors when storing cart items
      source: valkey-pod-1
      target: cart-service-1
    - condition: Checkout service receives errors when trying to access cart information
      effect: Checkout service experiences errors during cart operations
      source: cart-service-1
      target: checkout-service-1
    - condition: Frontend receives errors from checkout service due to cart failures
      effect: Frontend service experiences errors during cart and checkout operations
      source: checkout-service-1
      target: frontend-service-1
    - condition: Frontend-proxy receives errors from frontend service due to cart/checkout failures
      effect: Frontend-proxy service experiences errors when proxying requests
      source: frontend-service-1
      target: frontend-proxy-service-1
  recommendedActions:
    - solution:
        actions:
          - Identify and remove the storage-filler sidecar container from valkey-cart deployment
          - Delete the /data/filler file to free up disk space
          - Verify valkey-cart can successfully write data
          - Monitor disk usage to prevent future storage exhaustion
        id: fix_storage_issue


---
metadata:
  version: "v1"
fault:
  - entity:
      name: otel-demo-checkout-kafka-network-partition
      group_id: otel-demo-checkout-kafka-network-partition
      kind: NetworkChaos
    condition: Chaos Mesh network partition blocking checkout pods from reaching Kafka
    category: Network
    fault_mechanism: network_partition
alerts:
  - id: KafkaConnectionClosureRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service experiencing high Kafka connection closure rate due to network partition
  - id: RequestLatency
    group_id: checkout-service-1
    metadata:
      description: Checkout service experiencing increased latency due to Kafka connection issues
groups:
  - id: otel-demo-checkout-kafka-network-partition
    kind: NetworkChaos
    filter:
      - otel-demo-checkout-kafka-network-partition
    namespace: chaos-mesh
    root_cause: true
  - id: checkout-pod-1
    kind: Pod
    filter:
      - checkout-.*
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: kafka-service-1
    kind: Service
    filter:
      - kafka\b
    namespace: otel-demo
aliases:
  - - checkout-service-1
    - checkout-pod-1
propagations:
  - source: otel-demo-checkout-kafka-network-partition
    target: checkout-pod-1
    condition: Chaos Mesh NetworkChaos resource injects network partition affecting checkout pods
    effect: Checkout pods cannot establish or maintain connections to Kafka
  - source: checkout-pod-1
    target: checkout-service-1
    condition: Checkout pods cannot communicate with Kafka due to network partition
    effect: Checkout service cannot publish order events to Kafka, but continues processing orders
  - source: checkout-service-1
    target: kafka-service-1
    condition: Network partition prevents checkout service from communicating with Kafka
    effect: Order events are not published to Kafka queue, downstream services miss order notifications
recommended_actions:
  - solution:
      id: resolve_network_partition
      actions:
        - Identify and resolve network partition between checkout and Kafka pods
        - Verify network policies and connectivity between the services


---
metadata:
  version: "v1"
fault:
  - entity:
      name: otel-demo-shipping-quote-network-partition
      group_id: otel-demo-shipping-quote-network-partition
      kind: NetworkChaos
    condition: Chaos Mesh network partition blocking shipping pods from reaching quote service
    category: Network
    fault_mechanism: network_partition
alerts:
  - id: RequestErrorRate
    group_id: shipping-service-1
    metadata:
      description: Shipping service cannot reach quote service due to network partition, returning 500 errors
  - id: RequestLatency
    group_id: shipping-service-1
    metadata:
      description: Shipping service experiencing increased latency due to quote service timeouts
  - id: RequestErrorRate
    group_id: checkout-service-1
    metadata:
      description: Checkout service receiving errors from shipping service
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Frontend service experiencing errors during checkout flow
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Frontend proxy experiencing errors from failed checkout attempts
groups:
  - id: otel-demo-shipping-quote-network-partition
    kind: NetworkChaos
    filter:
      - otel-demo-shipping-quote-network-partition
    namespace: chaos-mesh
    root_cause: true
  - id: shipping-pod-1
    kind: Pod
    filter:
      - shipping-.*
    namespace: otel-demo
  - id: shipping-service-1
    kind: Service
    filter:
      - shipping\b
    namespace: otel-demo
  - id: quote-service-1
    kind: Service
    filter:
      - quote\b
    namespace: otel-demo
  - id: checkout-service-1
    kind: Service
    filter:
      - checkout\b
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
aliases:
  - - shipping-service-1
    - shipping-pod-1
propagations:
  - source: otel-demo-shipping-quote-network-partition
    target: shipping-pod-1
    condition: Chaos Mesh NetworkChaos resource injects network partition affecting shipping pods
    effect: Shipping pods cannot establish or maintain connections to quote service
  - source: shipping-pod-1
    target: shipping-service-1
    condition: Shipping pods cannot communicate with quote service due to network partition
    effect: Shipping service cannot retrieve shipping cost quotes and returns 500 errors
  - source: shipping-service-1
    target: quote-service-1
    condition: Network partition prevents shipping service from communicating with quote service
    effect: Quote service becomes unreachable from shipping, causing shipping requests to fail
  - source: quote-service-1
    target: checkout-service-1
    condition: Checkout service calls shipping service which fails due to quote service unavailability
    effect: Checkout operations fail when attempting to calculate shipping costs
  - source: checkout-service-1
    target: frontend-service-1
    condition: Frontend receives errors from checkout when shipping calculations fail
    effect: Users cannot complete checkout due to shipping service failures
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend returns errors to proxy for failed checkout attempts
    effect: User-facing checkout operations fail through the proxy
recommended_actions:
  - solution:
      id: resolve_network_partition
      actions:
        - Identify and resolve network partition between shipping and quote pods
        - Verify network policies and connectivity between the services


---
metadata:
  version: "v1"
alerts:
  - group_id: checkout-service-1
    id: RequestLatency
    metadata:
      description: Checkout service experiencing increased latency due to email service connection timeouts
aliases:
  - - email-service-1
    - email-pod-1
fault:
  - category: Network
    condition: Chaos Mesh network partition blocking email pods from reaching checkout service
    entity:
      group_id: otel-demo-email-checkout-network-partition
      kind: NetworkChaos
      name: otel-demo-email-checkout-network-partition
    fault_mechanism: network_partition
groups:
  - filter:
      - otel-demo-email-checkout-network-partition
    id: otel-demo-email-checkout-network-partition
    kind: NetworkChaos
    namespace: chaos-mesh
    root_cause: true
  - filter:
      - checkout-.*
    id: checkout-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - email-.*
    id: email-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - email\b
    id: email-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - checkout\b
    id: checkout-service-1
    kind: Service
    namespace: otel-demo
propagations:
  - condition: Chaos Mesh NetworkChaos resource injects network partition affecting email and checkout pods
    effect: Email pods cannot receive connections from checkout service
    source: otel-demo-email-checkout-network-partition
    target: email-pod-1
  - condition: Chaos Mesh NetworkChaos resource injects network partition affecting email and checkout pods
    effect: Checkout pods cannot receive connections
    source: otel-demo-email-checkout-network-partition
    target: checkout-pod-1
  - condition: Email pods cannot receive or respond to checkout requests due to network partition
    effect: Email service becomes unreachable from checkout service
    source: email-pod-1
    target: email-service-1
  - condition: Checkout service cannot connect to email service to send order confirmation emails
    effect: Checkout completes orders but experiences latency waiting for email service timeouts, no confirmation emails sent
    source: email-service-1
    target: checkout-service-1
recommended_actions:
  - solution:
      actions:
        - Identify and stop the network partition chaos experiment between email and checkout
      id: resolve_network_partition


---
metadata:
  version: "v1"
alerts:
  - group_id: otel-demo-fraud-detection-kafka-network-partition
    id: KafkaConnectionClosureRate
    metadata:
      description: Fraud detection service experiencing high Kafka connection closure rate due to network partition
  - group_id: fraud-detection-service-1
    id: RequestErrorRate
    metadata:
      description: Fraud detection service unable to publish or consume messages from Kafka
aliases:
  - - fraud-detection-service-1
    - fraud-detection-pod-1
fault:
  - category: Network
    condition: Chaos Mesh network partition blocking fraud-detection pods from reaching Kafka
    entity:
      group_id: otel-demo-fraud-detection-kafka-network-partition
      kind: NetworkChaos
      name: otel-demo-fraud-detection-kafka-network-partition
    fault_mechanism: network_partition
groups:
  - filter:
      - otel-demo-fraud-detection-kafka-network-partition
    id: otel-demo-fraud-detection-kafka-network-partition
    kind: NetworkChaos
    namespace: chaos-mesh
    root_cause: true
  - filter:
      - fraud-detection-.*
    id: fraud-detection-pod-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - kafka-.*
    id: kafka-1
    kind: Pod
    namespace: otel-demo
  - filter:
      - fraud-detection\b
    id: fraud-detection-service-1
    kind: Service
    namespace: otel-demo
  - filter:
      - kafka\b
    id: kafka-service-1
    kind: Service
    namespace: otel-demo
propagations:
  - condition: Chaos Mesh NetworkChaos resource injects network partition affecting fraud-detection and kafka pods
    effect: Fraud detection pods cannot establish or maintain connections to Kafka
    source: otel-demo-fraud-detection-kafka-network-partition
    target: fraud-detection-pod-1
  - condition: Chaos Mesh NetworkChaos resource injects network partition affecting fraud-detection and kafka pods
    effect: Kafka pods cannot establish or maintain connections
    source: otel-demo-fraud-detection-kafka-network-partition
    target: kafka-pod-1
  - condition: Fraud detection pods cannot communicate with Kafka due to network partition
    effect: Fraud detection service cannot consume order events from Kafka or publish fraud analysis results
    source: fraud-detection-pod-1
    target: fraud-detection-service-1
  - condition: Network partition prevents fraud detection service from communicating with Kafka
    effect: Order fraud analysis pipeline is blocked, fraud checks not performed on new orders
    source: fraud-detection-service-1
    target: kafka-service-1
recommended_actions:
  - solution:
      actions:
        - Identify and resolve network partition between fraud-detection and Kafka pods
        - Verify network policies and connectivity between the services
      id: resolve_network_partition


---
fault:
  - entity:
      name: otel-demo-namespace-1
      group_id: otel-demo-namespace-1
      kind: Namespace
    condition: Memory quota imposed on namespace preventing pod scheduling
    category: custom
    fault_mechanism: resource_quota_enforcement
alerts:
  - id: NoRequestsDetected
    group_id: ad-service-1
    metadata:
      description: Service ad in namespace otel-demo has 0 requests
  - id: RequestErrorRate
    group_id: frontend-service-1
    metadata:
      description: Request error rate in service frontend in namespace otel-demo is above 0
  - id: RequestErrorRate
    group_id: frontend-proxy-service-1
    metadata:
      description: Request error rate in service frontend-proxy in namespace otel-demo is above 0
groups:
  - id: otel-demo-namespace-1
    kind: Namespace
    filter:
      - otel-demo
    root_cause: true
  - id: ad-pod-1
    kind: Pod
    filter:
      - ad-.*
    namespace: otel-demo
  - id: ad-service-1
    kind: Service
    filter:
      - ad\b
    namespace: otel-demo
  - id: ad-deployment-1
    kind: Deployment
    filter:
      - ad\b
    namespace: otel-demo
  - id: ad-replicaset-1
    kind: ReplicaSet
    filter:
      - ad-.*
    namespace: otel-demo
  - id: frontend-service-1
    kind: Service
    filter:
      - frontend\b
    namespace: otel-demo
  - id: frontend-pod-1
    kind: Pod
    filter:
      - frontend-.*
    namespace: otel-demo
  - id: frontend-proxy-service-1
    kind: Service
    filter:
      - frontend-proxy\b
    namespace: otel-demo
  - id: frontend-proxy-pod-1
    kind: Pod
    filter:
      - frontend-proxy-.*
    namespace: otel-demo
  - id: flagd-service-1
    kind: Service
    filter:
      - flagd\b
    namespace: otel-demo
  - id: flagd-pod-1
    kind: Pod
    filter:
      - flagd-.*
    namespace: otel-demo
aliases:
  - - ad-service-1
    - ad-pod-1
    - ad-deployment-1
    - ad-replicaset-1
  - - frontend-service-1
    - frontend-pod-1
  - - frontend-proxy-service-1
    - frontend-proxy-pod-1
propagations:
  - source: otel-demo-namespace-1
    target: ad-deployment-1
    condition: Memory quota prevents new pod scheduling when deployment tries to scale up
    effect: Ad deployment unable to create new pods despite scale-up command
  - source: ad-deployment-1
    target: ad-replicaset-1
    condition: Deployment cannot create new ReplicaSet due to namespace memory constraints
    effect: Ad ReplicaSet shows 0/1 replicas created with FailedCreate condition
  - source: ad-replicaset-1
    target: ad-pod-1
    condition: ReplicaSet cannot schedule pods due to memory quota enforcement
    effect: Ad pods cannot be created, resulting in MinimumReplicasUnavailable
  - source: ad-pod-1
    target: ad-service-1
    condition: No healthy pods available to serve requests
    effect: Ad service becomes unavailable and receives no requests
  - source: ad-service-1
    target: frontend-service-1
    condition: Frontend cannot retrieve ads via gRPC call to ad service
    effect: Frontend service experiences request errors
  - source: frontend-service-1
    target: frontend-proxy-service-1
    condition: Frontend errors propagate to frontend-proxy due to service dependencies
    effect: Frontend-proxy service experiences request errors
recommended_actions:
  - solution:
      id: increase_memory_quota
      actions:
        - Evaluate current namespace memory usage and requirements
        - Increase memory quota on otel-demo namespace to accommodate all services


---
apiVersion: itbench.io/v1
kind: GroundTruth
metadata:
  name: scenario-105
spec:
  alerts:
    - group_id: product-catalog-pod-1
      id: PodCrashLoopBackOff
      metadata:
        description: Product catalog pod is in CrashLoopBackOff state
    - group_id: checkout-service-1
      id: RequestErrorRate
      metadata:
        description: Checkout service experiencing connection failures to product-catalog
    - group_id: payment-service-1
      id: NoRequestsDetected
      metadata:
        description: Payment service receiving no requests - checkout flow blocked
    - group_id: shipping-service-1
      id: NoRequestsDetected
      metadata:
        description: Shipping service receiving no requests - checkout flow blocked
    - group_id: email-service-1
      id: NoRequestsDetected
      metadata:
        description: Email service receiving no requests - checkout flow blocked
    - group_id: currency-service-1
      id: NoRequestsDetected
      metadata:
        description: Currency service receiving no requests - checkout flow blocked
    - group_id: accounting-service-1
      id: NoRequestsDetected
      metadata:
        description: Accounting service receiving no requests - order processing blocked
    - group_id: fraud-detection-service-1
      id: NoRequestsDetected
      metadata:
        description: Fraud detection service receiving no requests - order processing blocked
  aliases:
    - - product-catalog-service-1
      - product-catalog-pod-1
    - - checkout-service-1
      - checkout-pod-1
    - - payment-service-1
      - payment-pod-1
    - - shipping-service-1
      - shipping-pod-1
    - - email-service-1
      - email-pod-1
    - - currency-service-1
      - currency-pod-1
    - - accounting-service-1
      - accounting-pod-1
    - - fraud-detection-service-1
      - fraud-detection-pod-1
    - - quote-service-1
      - quote-pod-1
  fault:
    - category: Infrastructure
      condition: Pod container in CrashLoopBackOff state due to invalid command
      entity:
        group_id: product-catalog-pod-1
        kind: Pod
        name: product-catalog-pod-1
      fault_mechanism: command_injection
  groups:
    - filter:
        - product-catalog-.*
      id: product-catalog-deployment
      kind: Deployment
      namespace: otel-demo
      root_cause: true
    - filter:
        - product-catalog-.*
      id: product-catalog-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - product-catalog\b
      id: product-catalog-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - checkout\b
      id: checkout-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - checkout-.*
      id: checkout-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - payment\b
      id: payment-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - payment-.*
      id: payment-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - shipping\b
      id: shipping-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - shipping-.*
      id: shipping-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - email\b
      id: email-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - email-.*
      id: email-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - currency\b
      id: currency-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - currency-.*
      id: currency-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - accounting\b
      id: accounting-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - accounting-.*
      id: accounting-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - fraud-detection\b
      id: fraud-detection-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - fraud-detection-.*
      id: fraud-detection-pod-1
      kind: Pod
      namespace: otel-demo
    - filter:
        - quote\b
      id: quote-service-1
      kind: Service
      namespace: otel-demo
    - filter:
        - quote-.*
      id: quote-pod-1
      kind: Pod
      namespace: otel-demo
  propagations:
    - condition: Invalid command in deployment
      effect: Pod container fails to start due to invalid command
      source: product-catalog-deployment
      target: product-catalog-pod-1
    - condition: Pod container fails to start due to invalid command
      effect: Product catalog service becomes unavailable
      source: product-catalog-pod-1
      target: product-catalog-service-1
    - condition: Checkout service cannot validate products via product-catalog
      effect: Checkout operations fail early in the process
      source: product-catalog-service-1
      target: checkout-service-1
    - condition: Checkout process never reaches payment stage due to product validation failure
      effect: Payment service receives zero requests
      source: checkout-service-1
      target: payment-service-1
    - condition: Checkout process never reaches shipping stage due to product validation failure
      effect: Shipping service receives zero requests
      source: checkout-service-1
      target: shipping-service-1
    - condition: Checkout process never completes to send order confirmation emails
      effect: Email service receives zero requests
      source: checkout-service-1
      target: email-service-1
    - condition: Checkout process never reaches currency conversion stage
      effect: Currency service receives zero requests
      source: checkout-service-1
      target: currency-service-1
    - condition: No completed orders to process via Kafka queue
      effect: Accounting service receives zero order messages
      source: checkout-service-1
      target: accounting-service-1
    - condition: No completed orders to analyze via Kafka queue
      effect: Fraud detection service receives zero order messages
      source: checkout-service-1
      target: fraud-detection-service-1
    - condition: Shipping service not processing any requests
      effect: Quote service receives zero shipping quote requests
      source: shipping-service-1
      target: quote-service-1
  recommended_actions:
    - solution:
        actions:
          - Identify and correct the invalid command in product-catalog deployment
        id: fix_container_command