{
  "date": "2026-09-29T11:29:36.093561+00:00",
  "python": "3.12.14",
  "laya": "0.3.20",
  "checkpoint_selection": "english",
  "note": "First call can include download/loading. Eight synthetic cases are not a benchmark.",
  "results": [
    {
      "id": 1,
      "text": "I was charged twice for my subscription.",
      "expected": "billing",
      "elapsed_seconds": 59.129652300005546,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "billing",
            "probabilities": {
              "billing": 0.9857,
              "technical": 0.0046,
              "sales": 0.0051,
              "other": 0.0046
            },
            "confidence": 0.9348,
            "answer_confidence": 0.9857,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 58,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 2,
      "text": "The app crashes whenever I open settings.",
      "expected": "technical",
      "elapsed_seconds": 0.28916500000923406,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "technical",
            "probabilities": {
              "billing": 0.0185,
              "technical": 0.9303,
              "sales": 0.0251,
              "other": 0.0261
            },
            "confidence": 0.7628,
            "answer_confidence": 0.9303,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 58,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 3,
      "text": "Can you send a quote for twenty seats?",
      "expected": "sales",
      "elapsed_seconds": 0.26928310000221245,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "sales",
            "probabilities": {
              "billing": 0.1042,
              "technical": 0.0539,
              "sales": 0.7341,
              "other": 0.1078
            },
            "confidence": 0.3795,
            "answer_confidence": 0.7341,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 59,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 4,
      "text": "Thanks, your team was helpful.",
      "expected": "other",
      "elapsed_seconds": 0.2605634000065038,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "other",
            "probabilities": {
              "billing": 0.299,
              "technical": 0.1576,
              "sales": 0.2111,
              "other": 0.3323
            },
            "confidence": 0.0286,
            "answer_confidence": 0.3323,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 57,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 5,
      "text": "Please send my invoice for last month.",
      "expected": "billing",
      "elapsed_seconds": 0.2616022000001976,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "billing",
            "probabilities": {
              "billing": 0.9911,
              "technical": 0.0029,
              "sales": 0.0027,
              "other": 0.0033
            },
            "confidence": 0.9562,
            "answer_confidence": 0.9911,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 58,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 6,
      "text": "My reset-password link returns a server error.",
      "expected": "technical",
      "elapsed_seconds": 0.2681575000024168,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "technical",
            "probabilities": {
              "billing": 0.0343,
              "technical": 0.8888,
              "sales": 0.0336,
              "other": 0.0433
            },
            "confidence": 0.6607,
            "answer_confidence": 0.8888,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 60,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 7,
      "text": "Do you offer a demonstration before we purchase?",
      "expected": "sales",
      "elapsed_seconds": 0.2583162000082666,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "sales",
            "probabilities": {
              "billing": 0.0956,
              "technical": 0.1533,
              "sales": 0.5232,
              "other": 0.2279
            },
            "confidence": 0.1431,
            "answer_confidence": 0.5232,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 59,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    },
    {
      "id": 8,
      "text": "I need some help.",
      "expected": "other",
      "elapsed_seconds": 0.24096360000839923,
      "raw_result": {
        "model": "laya-rl-agent",
        "answers": {
          "department": {
            "type": "choice",
            "choice": "other",
            "probabilities": {
              "billing": 0.2107,
              "technical": 0.2338,
              "sales": 0.2134,
              "other": 0.3422
            },
            "confidence": 0.0158,
            "answer_confidence": 0.3422,
            "action": {
              "act_probability": 1
            }
          }
        },
        "usage": {
          "input_tokens": 55,
          "output_tokens": 0
        },
        "routing": {
          "model": "english",
          "repo": "convaiinnovations/laya",
          "reason": "explicit model='english'",
          "detection": null,
          "workflow": null
        }
      }
    }
  ],
  "environment": {
    "os": "Windows",
    "cpu": "Intel Core i5-1335U",
    "device": "CPU",
    "torch": "2.14.0+cpu",
    "model_revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
  },
  "warnings": [
    "SDK warned that the checkpoint temperature for choice:11+ was outside its allowed range and clamped it to 0.5; affected confidence values should be treated as uncalibrated. This test used four choices and did not evaluate calibration."
  ]
}