| { | |
| "metadata": { | |
| "checkpoint": "checkpoints/c1_best.pt", | |
| "onnx_model": "checkpoints/c1_decision_jef.onnx", | |
| "dataset_sha256": "0e0c6c5bd41d8b748943c0d1300731b273499b472f7ca091a88be44677f2d682", | |
| "total_cases": 525, | |
| "parity_fixed": true, | |
| "date": "2026-09-29 08:58:43" | |
| }, | |
| "parity": { | |
| "pyt_vs_onnx_cpu_mismatches": 0, | |
| "pyt_vs_onnx_cuda_mismatches": 0, | |
| "agreement_rate_cpu": 100.0, | |
| "agreement_rate_cuda": 100.0, | |
| "max_logit_diff": 0.000164031982421875, | |
| "mean_logit_diff": 1.9703918951563537e-05, | |
| "max_prob_diff": 1.0579824447631836e-05, | |
| "mean_prob_diff": 1.2620022005194187e-07 | |
| }, | |
| "order_stability": { | |
| "pytorch": { | |
| "reverse_flips": 8, | |
| "shuffled_flips": 6 | |
| }, | |
| "onnx_cpu": { | |
| "reverse_flips": 8, | |
| "shuffled_flips": 6 | |
| }, | |
| "onnx_cuda": { | |
| "reverse_flips": 8, | |
| "shuffled_flips": 6 | |
| } | |
| }, | |
| "metrics": { | |
| "pytorch": { | |
| "total": 525, | |
| "correct": 485, | |
| "accuracy": 0.9238, | |
| "macro_f1": 0.9161, | |
| "micro_f1": 0.9238, | |
| "per_capability": { | |
| "ambiguous": { | |
| "support": 25, | |
| "tp": 14, | |
| "fp": 1, | |
| "fn": 11, | |
| "accuracy": 0.56, | |
| "precision": 0.9333, | |
| "recall": 0.56, | |
| "f1": 0.7 | |
| }, | |
| "campaigns.send": { | |
| "support": 35, | |
| "tp": 31, | |
| "fp": 1, | |
| "fn": 4, | |
| "accuracy": 0.8857, | |
| "precision": 0.9688, | |
| "recall": 0.8857, | |
| "f1": 0.9254 | |
| }, | |
| "catalog.search": { | |
| "support": 42, | |
| "tp": 37, | |
| "fp": 1, | |
| "fn": 5, | |
| "accuracy": 0.881, | |
| "precision": 0.9737, | |
| "recall": 0.881, | |
| "f1": 0.925 | |
| }, | |
| "customers.search": { | |
| "support": 38, | |
| "tp": 34, | |
| "fp": 2, | |
| "fn": 4, | |
| "accuracy": 0.8947, | |
| "precision": 0.9444, | |
| "recall": 0.8947, | |
| "f1": 0.9189 | |
| }, | |
| "invoices.get": { | |
| "support": 38, | |
| "tp": 36, | |
| "fp": 0, | |
| "fn": 2, | |
| "accuracy": 0.9474, | |
| "precision": 1.0, | |
| "recall": 0.9474, | |
| "f1": 0.973 | |
| }, | |
| "messages.send": { | |
| "support": 35, | |
| "tp": 33, | |
| "fp": 4, | |
| "fn": 2, | |
| "accuracy": 0.9429, | |
| "precision": 0.8919, | |
| "recall": 0.9429, | |
| "f1": 0.9167 | |
| }, | |
| "orders.cancel": { | |
| "support": 43, | |
| "tp": 40, | |
| "fp": 7, | |
| "fn": 3, | |
| "accuracy": 0.9302, | |
| "precision": 0.8511, | |
| "recall": 0.9302, | |
| "f1": 0.8889 | |
| }, | |
| "orders.list": { | |
| "support": 47, | |
| "tp": 46, | |
| "fp": 5, | |
| "fn": 1, | |
| "accuracy": 0.9787, | |
| "precision": 0.902, | |
| "recall": 0.9787, | |
| "f1": 0.9388 | |
| }, | |
| "orders.refund": { | |
| "support": 41, | |
| "tp": 38, | |
| "fp": 0, | |
| "fn": 3, | |
| "accuracy": 0.9268, | |
| "precision": 1.0, | |
| "recall": 0.9268, | |
| "f1": 0.962 | |
| }, | |
| "orders.status": { | |
| "support": 44, | |
| "tp": 40, | |
| "fp": 0, | |
| "fn": 4, | |
| "accuracy": 0.9091, | |
| "precision": 1.0, | |
| "recall": 0.9091, | |
| "f1": 0.9524 | |
| }, | |
| "records.delete": { | |
| "support": 35, | |
| "tp": 35, | |
| "fp": 0, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0 | |
| }, | |
| "sales.summary": { | |
| "support": 35, | |
| "tp": 35, | |
| "fp": 1, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 0.9722, | |
| "recall": 1.0, | |
| "f1": 0.9859 | |
| }, | |
| "tickets.create": { | |
| "support": 38, | |
| "tp": 38, | |
| "fp": 5, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 0.8837, | |
| "recall": 1.0, | |
| "f1": 0.9383 | |
| }, | |
| "unknown": { | |
| "support": 29, | |
| "tp": 28, | |
| "fp": 13, | |
| "fn": 1, | |
| "accuracy": 0.9655, | |
| "precision": 0.6829, | |
| "recall": 0.9655, | |
| "f1": 0.8 | |
| } | |
| }, | |
| "confusion_matrix": { | |
| "ambiguous": { | |
| "ambiguous": 14, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 5, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 5 | |
| }, | |
| "campaigns.send": { | |
| "ambiguous": 0, | |
| "campaigns.send": 31, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 4, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "catalog.search": { | |
| "ambiguous": 0, | |
| "campaigns.send": 1, | |
| "catalog.search": 37, | |
| "customers.search": 2, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 2 | |
| }, | |
| "customers.search": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 34, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 2, | |
| "unknown": 1 | |
| }, | |
| "invoices.get": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 36, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 2, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "messages.send": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 33, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 1 | |
| }, | |
| "orders.cancel": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 40, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 2 | |
| }, | |
| "orders.list": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 46, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 1, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "orders.refund": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 1, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 1, | |
| "orders.list": 0, | |
| "orders.refund": 38, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 1 | |
| }, | |
| "orders.status": { | |
| "ambiguous": 1, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 1, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 40, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 1 | |
| }, | |
| "records.delete": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 35, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "sales.summary": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 35, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "tickets.create": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 38, | |
| "unknown": 0 | |
| }, | |
| "unknown": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 28 | |
| } | |
| } | |
| }, | |
| "onnx_cpu": { | |
| "total": 525, | |
| "correct": 485, | |
| "accuracy": 0.9238, | |
| "macro_f1": 0.9161, | |
| "micro_f1": 0.9238, | |
| "per_capability": { | |
| "ambiguous": { | |
| "support": 25, | |
| "tp": 14, | |
| "fp": 1, | |
| "fn": 11, | |
| "accuracy": 0.56, | |
| "precision": 0.9333, | |
| "recall": 0.56, | |
| "f1": 0.7 | |
| }, | |
| "campaigns.send": { | |
| "support": 35, | |
| "tp": 31, | |
| "fp": 1, | |
| "fn": 4, | |
| "accuracy": 0.8857, | |
| "precision": 0.9688, | |
| "recall": 0.8857, | |
| "f1": 0.9254 | |
| }, | |
| "catalog.search": { | |
| "support": 42, | |
| "tp": 37, | |
| "fp": 1, | |
| "fn": 5, | |
| "accuracy": 0.881, | |
| "precision": 0.9737, | |
| "recall": 0.881, | |
| "f1": 0.925 | |
| }, | |
| "customers.search": { | |
| "support": 38, | |
| "tp": 34, | |
| "fp": 2, | |
| "fn": 4, | |
| "accuracy": 0.8947, | |
| "precision": 0.9444, | |
| "recall": 0.8947, | |
| "f1": 0.9189 | |
| }, | |
| "invoices.get": { | |
| "support": 38, | |
| "tp": 36, | |
| "fp": 0, | |
| "fn": 2, | |
| "accuracy": 0.9474, | |
| "precision": 1.0, | |
| "recall": 0.9474, | |
| "f1": 0.973 | |
| }, | |
| "messages.send": { | |
| "support": 35, | |
| "tp": 33, | |
| "fp": 4, | |
| "fn": 2, | |
| "accuracy": 0.9429, | |
| "precision": 0.8919, | |
| "recall": 0.9429, | |
| "f1": 0.9167 | |
| }, | |
| "orders.cancel": { | |
| "support": 43, | |
| "tp": 40, | |
| "fp": 7, | |
| "fn": 3, | |
| "accuracy": 0.9302, | |
| "precision": 0.8511, | |
| "recall": 0.9302, | |
| "f1": 0.8889 | |
| }, | |
| "orders.list": { | |
| "support": 47, | |
| "tp": 46, | |
| "fp": 5, | |
| "fn": 1, | |
| "accuracy": 0.9787, | |
| "precision": 0.902, | |
| "recall": 0.9787, | |
| "f1": 0.9388 | |
| }, | |
| "orders.refund": { | |
| "support": 41, | |
| "tp": 38, | |
| "fp": 0, | |
| "fn": 3, | |
| "accuracy": 0.9268, | |
| "precision": 1.0, | |
| "recall": 0.9268, | |
| "f1": 0.962 | |
| }, | |
| "orders.status": { | |
| "support": 44, | |
| "tp": 40, | |
| "fp": 0, | |
| "fn": 4, | |
| "accuracy": 0.9091, | |
| "precision": 1.0, | |
| "recall": 0.9091, | |
| "f1": 0.9524 | |
| }, | |
| "records.delete": { | |
| "support": 35, | |
| "tp": 35, | |
| "fp": 0, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0 | |
| }, | |
| "sales.summary": { | |
| "support": 35, | |
| "tp": 35, | |
| "fp": 1, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 0.9722, | |
| "recall": 1.0, | |
| "f1": 0.9859 | |
| }, | |
| "tickets.create": { | |
| "support": 38, | |
| "tp": 38, | |
| "fp": 5, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 0.8837, | |
| "recall": 1.0, | |
| "f1": 0.9383 | |
| }, | |
| "unknown": { | |
| "support": 29, | |
| "tp": 28, | |
| "fp": 13, | |
| "fn": 1, | |
| "accuracy": 0.9655, | |
| "precision": 0.6829, | |
| "recall": 0.9655, | |
| "f1": 0.8 | |
| } | |
| }, | |
| "confusion_matrix": { | |
| "ambiguous": { | |
| "ambiguous": 14, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 5, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 5 | |
| }, | |
| "campaigns.send": { | |
| "ambiguous": 0, | |
| "campaigns.send": 31, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 4, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "catalog.search": { | |
| "ambiguous": 0, | |
| "campaigns.send": 1, | |
| "catalog.search": 37, | |
| "customers.search": 2, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 2 | |
| }, | |
| "customers.search": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 34, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 2, | |
| "unknown": 1 | |
| }, | |
| "invoices.get": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 36, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 2, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "messages.send": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 33, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 1 | |
| }, | |
| "orders.cancel": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 40, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 2 | |
| }, | |
| "orders.list": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 46, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 1, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "orders.refund": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 1, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 1, | |
| "orders.list": 0, | |
| "orders.refund": 38, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 1 | |
| }, | |
| "orders.status": { | |
| "ambiguous": 1, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 1, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 40, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 1 | |
| }, | |
| "records.delete": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 35, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "sales.summary": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 35, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "tickets.create": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 38, | |
| "unknown": 0 | |
| }, | |
| "unknown": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 28 | |
| } | |
| } | |
| }, | |
| "onnx_cuda": { | |
| "total": 525, | |
| "correct": 485, | |
| "accuracy": 0.9238, | |
| "macro_f1": 0.9161, | |
| "micro_f1": 0.9238, | |
| "per_capability": { | |
| "ambiguous": { | |
| "support": 25, | |
| "tp": 14, | |
| "fp": 1, | |
| "fn": 11, | |
| "accuracy": 0.56, | |
| "precision": 0.9333, | |
| "recall": 0.56, | |
| "f1": 0.7 | |
| }, | |
| "campaigns.send": { | |
| "support": 35, | |
| "tp": 31, | |
| "fp": 1, | |
| "fn": 4, | |
| "accuracy": 0.8857, | |
| "precision": 0.9688, | |
| "recall": 0.8857, | |
| "f1": 0.9254 | |
| }, | |
| "catalog.search": { | |
| "support": 42, | |
| "tp": 37, | |
| "fp": 1, | |
| "fn": 5, | |
| "accuracy": 0.881, | |
| "precision": 0.9737, | |
| "recall": 0.881, | |
| "f1": 0.925 | |
| }, | |
| "customers.search": { | |
| "support": 38, | |
| "tp": 34, | |
| "fp": 2, | |
| "fn": 4, | |
| "accuracy": 0.8947, | |
| "precision": 0.9444, | |
| "recall": 0.8947, | |
| "f1": 0.9189 | |
| }, | |
| "invoices.get": { | |
| "support": 38, | |
| "tp": 36, | |
| "fp": 0, | |
| "fn": 2, | |
| "accuracy": 0.9474, | |
| "precision": 1.0, | |
| "recall": 0.9474, | |
| "f1": 0.973 | |
| }, | |
| "messages.send": { | |
| "support": 35, | |
| "tp": 33, | |
| "fp": 4, | |
| "fn": 2, | |
| "accuracy": 0.9429, | |
| "precision": 0.8919, | |
| "recall": 0.9429, | |
| "f1": 0.9167 | |
| }, | |
| "orders.cancel": { | |
| "support": 43, | |
| "tp": 40, | |
| "fp": 7, | |
| "fn": 3, | |
| "accuracy": 0.9302, | |
| "precision": 0.8511, | |
| "recall": 0.9302, | |
| "f1": 0.8889 | |
| }, | |
| "orders.list": { | |
| "support": 47, | |
| "tp": 46, | |
| "fp": 5, | |
| "fn": 1, | |
| "accuracy": 0.9787, | |
| "precision": 0.902, | |
| "recall": 0.9787, | |
| "f1": 0.9388 | |
| }, | |
| "orders.refund": { | |
| "support": 41, | |
| "tp": 38, | |
| "fp": 0, | |
| "fn": 3, | |
| "accuracy": 0.9268, | |
| "precision": 1.0, | |
| "recall": 0.9268, | |
| "f1": 0.962 | |
| }, | |
| "orders.status": { | |
| "support": 44, | |
| "tp": 40, | |
| "fp": 0, | |
| "fn": 4, | |
| "accuracy": 0.9091, | |
| "precision": 1.0, | |
| "recall": 0.9091, | |
| "f1": 0.9524 | |
| }, | |
| "records.delete": { | |
| "support": 35, | |
| "tp": 35, | |
| "fp": 0, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 1.0, | |
| "recall": 1.0, | |
| "f1": 1.0 | |
| }, | |
| "sales.summary": { | |
| "support": 35, | |
| "tp": 35, | |
| "fp": 1, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 0.9722, | |
| "recall": 1.0, | |
| "f1": 0.9859 | |
| }, | |
| "tickets.create": { | |
| "support": 38, | |
| "tp": 38, | |
| "fp": 5, | |
| "fn": 0, | |
| "accuracy": 1.0, | |
| "precision": 0.8837, | |
| "recall": 1.0, | |
| "f1": 0.9383 | |
| }, | |
| "unknown": { | |
| "support": 29, | |
| "tp": 28, | |
| "fp": 13, | |
| "fn": 1, | |
| "accuracy": 0.9655, | |
| "precision": 0.6829, | |
| "recall": 0.9655, | |
| "f1": 0.8 | |
| } | |
| }, | |
| "confusion_matrix": { | |
| "ambiguous": { | |
| "ambiguous": 14, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 5, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 5 | |
| }, | |
| "campaigns.send": { | |
| "ambiguous": 0, | |
| "campaigns.send": 31, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 4, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "catalog.search": { | |
| "ambiguous": 0, | |
| "campaigns.send": 1, | |
| "catalog.search": 37, | |
| "customers.search": 2, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 2 | |
| }, | |
| "customers.search": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 34, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 2, | |
| "unknown": 1 | |
| }, | |
| "invoices.get": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 36, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 2, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "messages.send": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 33, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 1 | |
| }, | |
| "orders.cancel": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 40, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 2 | |
| }, | |
| "orders.list": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 46, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 1, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "orders.refund": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 1, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 1, | |
| "orders.list": 0, | |
| "orders.refund": 38, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 1 | |
| }, | |
| "orders.status": { | |
| "ambiguous": 1, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 1, | |
| "orders.list": 1, | |
| "orders.refund": 0, | |
| "orders.status": 40, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 1 | |
| }, | |
| "records.delete": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 35, | |
| "sales.summary": 0, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "sales.summary": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 35, | |
| "tickets.create": 0, | |
| "unknown": 0 | |
| }, | |
| "tickets.create": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 38, | |
| "unknown": 0 | |
| }, | |
| "unknown": { | |
| "ambiguous": 0, | |
| "campaigns.send": 0, | |
| "catalog.search": 0, | |
| "customers.search": 0, | |
| "invoices.get": 0, | |
| "messages.send": 0, | |
| "orders.cancel": 0, | |
| "orders.list": 0, | |
| "orders.refund": 0, | |
| "orders.status": 0, | |
| "records.delete": 0, | |
| "sales.summary": 0, | |
| "tickets.create": 1, | |
| "unknown": 28 | |
| } | |
| } | |
| } | |
| }, | |
| "benchmarks": { | |
| "cpu": { | |
| "infer_only": { | |
| "p50_ms": 152.0, | |
| "p95_ms": 217.7, | |
| "p99_ms": 235.33, | |
| "mean_ms": 161.58, | |
| "throughput_qps": 6.2 | |
| }, | |
| "end_to_end": { | |
| "p50_ms": 153.56, | |
| "p95_ms": 221.7, | |
| "p99_ms": 230.36, | |
| "mean_ms": 163.24, | |
| "throughput_qps": 6.1 | |
| }, | |
| "peak_rss_mb": 7723.7 | |
| }, | |
| "gpu": { | |
| "infer_only": { | |
| "p50_ms": 8.94, | |
| "p95_ms": 9.19, | |
| "p99_ms": 10.81, | |
| "mean_ms": 8.97, | |
| "throughput_qps": 111.4 | |
| }, | |
| "end_to_end": { | |
| "p50_ms": 9.5, | |
| "p95_ms": 9.98, | |
| "p99_ms": 11.56, | |
| "mean_ms": 9.54, | |
| "throughput_qps": 104.9 | |
| }, | |
| "peak_vram_mb": 0.0 | |
| } | |
| } | |
| } |