Spaces:
Running
Running
| <html lang="en" class="no-js"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <meta name="description" content="Validation intelligence for CI — failure-signature RCA, ML flaky-vs-real classification, and attested test-selection with a verifiable confidence bound."> | |
| <link rel="canonical" href="https://amitpatole.github.io/assaylab/api/"> | |
| <link rel="prev" href="../evaluation/"> | |
| <link rel="icon" href="../assets/images/favicon.png"> | |
| <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6"> | |
| <title>API - assaylab</title> | |
| <link rel="stylesheet" href="../assets/stylesheets/main.484c7ddc.min.css"> | |
| <link rel="stylesheet" href="../assets/stylesheets/palette.ab4e12ef.min.css"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Inter:300,300i,400,400i,700,700i%7CJetBrains+Mono:400,400i,700,700i&display=fallback"> | |
| <style>:root{--md-text-font:"Inter";--md-code-font:"JetBrains Mono"}</style> | |
| <link rel="stylesheet" href="../assets/_mkdocstrings.css"> | |
| <link rel="stylesheet" href="../stylesheets/extra.css"> | |
| <script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script> | |
| </head> | |
| <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo"> | |
| <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off"> | |
| <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off"> | |
| <label class="md-overlay" for="__drawer"></label> | |
| <div data-md-component="skip"> | |
| <a href="#api-reference" class="md-skip"> | |
| Skip to content | |
| </a> | |
| </div> | |
| <div data-md-component="announce"> | |
| </div> | |
| <header class="md-header md-header--shadow" data-md-component="header"> | |
| <nav class="md-header__inner md-grid" aria-label="Header"> | |
| <a href=".." title="assaylab" class="md-header__button md-logo" aria-label="assaylab" data-md-component="logo"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg> | |
| </a> | |
| <label class="md-header__button md-icon" for="__drawer"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg> | |
| </label> | |
| <div class="md-header__title" data-md-component="header-title"> | |
| <div class="md-header__ellipsis"> | |
| <div class="md-header__topic"> | |
| <span class="md-ellipsis"> | |
| assaylab | |
| </span> | |
| </div> | |
| <div class="md-header__topic" data-md-component="header-topic"> | |
| <span class="md-ellipsis"> | |
| API | |
| </span> | |
| </div> | |
| </div> | |
| </div> | |
| <label class="md-header__button md-icon" for="__search"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg> | |
| </label> | |
| <div class="md-search" data-md-component="search" role="dialog"> | |
| <label class="md-search__overlay" for="__search"></label> | |
| <div class="md-search__inner" role="search"> | |
| <form class="md-search__form" name="search"> | |
| <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required> | |
| <label class="md-search__icon md-icon" for="__search"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg> | |
| </label> | |
| <nav class="md-search__options" aria-label="Search"> | |
| <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg> | |
| </button> | |
| </nav> | |
| <div class="md-search__suggest" data-md-component="search-suggest"></div> | |
| </form> | |
| <div class="md-search__output"> | |
| <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix> | |
| <div class="md-search-result" data-md-component="search-result"> | |
| <div class="md-search-result__meta"> | |
| Initializing search | |
| </div> | |
| <ol class="md-search-result__list" role="presentation"></ol> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="md-header__source"> | |
| <a href="https://github.com/amitpatole/assaylab" title="Go to repository" class="md-source" data-md-component="source"> | |
| <div class="md-source__icon md-icon"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M173.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M252.8 8C114.1 8 8 113.3 8 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C436.2 457.8 504 362.9 504 252 504 113.3 391.5 8 252.8 8M105.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg> | |
| </div> | |
| <div class="md-source__repository"> | |
| amitpatole/assaylab | |
| </div> | |
| </a> | |
| </div> | |
| </nav> | |
| </header> | |
| <div class="md-container" data-md-component="container"> | |
| <main class="md-main" data-md-component="main"> | |
| <div class="md-main__inner md-grid"> | |
| <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" > | |
| <div class="md-sidebar__scrollwrap"> | |
| <div class="md-sidebar__inner"> | |
| <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0"> | |
| <label class="md-nav__title" for="__drawer"> | |
| <a href=".." title="assaylab" class="md-nav__button md-logo" aria-label="assaylab" data-md-component="logo"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg> | |
| </a> | |
| assaylab | |
| </label> | |
| <div class="md-nav__source"> | |
| <a href="https://github.com/amitpatole/assaylab" title="Go to repository" class="md-source" data-md-component="source"> | |
| <div class="md-source__icon md-icon"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M173.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6m-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3m44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9M252.8 8C114.1 8 8 113.3 8 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C436.2 457.8 504 362.9 504 252 504 113.3 391.5 8 252.8 8M105.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1m-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7m32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1m-11.4-14.7c-1.6 1-1.6 3.6 0 5.9s4.3 3.3 5.6 2.3c1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2"/></svg> | |
| </div> | |
| <div class="md-source__repository"> | |
| amitpatole/assaylab | |
| </div> | |
| </a> | |
| </div> | |
| <ul class="md-nav__list" data-md-scrollfix> | |
| <li class="md-nav__item"> | |
| <a href=".." class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Home | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="../quickstart/" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Quickstart | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="../rca/" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Root-cause analysis | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="../selection/" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Attested selection | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="../llm/" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Test generation & self-healing | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="../evaluation/" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Evaluation | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item md-nav__item--active"> | |
| <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc"> | |
| <label class="md-nav__link md-nav__link--active" for="__toc"> | |
| <span class="md-ellipsis"> | |
| API | |
| </span> | |
| <span class="md-nav__icon md-icon"></span> | |
| </label> | |
| <a href="./" class="md-nav__link md-nav__link--active"> | |
| <span class="md-ellipsis"> | |
| API | |
| </span> | |
| </a> | |
| <nav class="md-nav md-nav--secondary" aria-label="Table of contents"> | |
| <label class="md-nav__title" for="__toc"> | |
| <span class="md-nav__icon md-icon"></span> | |
| Table of contents | |
| </label> | |
| <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix> | |
| <li class="md-nav__item"> | |
| <a href="#models" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Models | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Models"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| models | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="models"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.Outcome" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Outcome | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.IssueKind" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| IssueKind | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.IssueSource" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| IssueSource | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.TestRecord" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| TestRecord | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.FailureSignature" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| FailureSignature | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="FailureSignature"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.FailureSignature.to_detail" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| to_detail | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.Issue" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Issue | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.Report" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Report | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#core-sense" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Core sense | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Core sense"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.analyze" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| analyze | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="analyze"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.analyze.grade_records" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| grade_records | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.analyze.analyze" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| analyze | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| signature | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="signature"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature.normalize" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| normalize | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature.fingerprint" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| fingerprint | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature.cluster" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| cluster | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#root-cause-analysis" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Root-cause analysis | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Root-cause analysis"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.rootcause" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| rootcause | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="rootcause"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.rootcause.categorize" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| categorize | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.flaky" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| flaky | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="flaky"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.flaky.flaky_probability" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| flaky_probability | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.risk" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| risk | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="risk"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.risk.rank_risk" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| rank_risk | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.model" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| model | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="model"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.model.train" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| train | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#attested-selection" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Attested selection | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Attested selection"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.select.engine" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| engine | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="engine"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.select.engine.select" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| select | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| receipt | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="receipt"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.Receipt" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Receipt | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Receipt"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.Receipt.signed_body" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| signed_body | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.Receipt.verify" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| verify | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.hash_ids" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| hash_ids | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.keys" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| keys | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="keys"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.keys.resolve_key" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| resolve_key | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.keys.key_id" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| key_id | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#test-generation-self-healing" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Test generation & self-healing | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Test generation & self-healing"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.models" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| models | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="models"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.models.Proposal" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Proposal | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Proposal"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.models.Proposal.signed_body" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| signed_body | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.generate" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| generate | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.gate" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| gate | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="gate"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.gate.evaluate_proposal" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| evaluate_proposal | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" > | |
| <div class="md-sidebar__scrollwrap"> | |
| <div class="md-sidebar__inner"> | |
| <nav class="md-nav md-nav--secondary" aria-label="Table of contents"> | |
| <label class="md-nav__title" for="__toc"> | |
| <span class="md-nav__icon md-icon"></span> | |
| Table of contents | |
| </label> | |
| <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix> | |
| <li class="md-nav__item"> | |
| <a href="#models" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Models | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Models"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| models | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="models"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.Outcome" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Outcome | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.IssueKind" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| IssueKind | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.IssueSource" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| IssueSource | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.TestRecord" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| TestRecord | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.FailureSignature" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| FailureSignature | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="FailureSignature"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.FailureSignature.to_detail" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| to_detail | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.Issue" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Issue | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.models.Report" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Report | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#core-sense" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Core sense | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Core sense"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.analyze" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| analyze | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="analyze"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.analyze.grade_records" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| grade_records | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.analyze.analyze" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| analyze | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| signature | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="signature"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature.normalize" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| normalize | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature.fingerprint" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| fingerprint | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.core.signature.cluster" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| cluster | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#root-cause-analysis" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Root-cause analysis | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Root-cause analysis"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.rootcause" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| rootcause | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="rootcause"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.rootcause.categorize" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| categorize | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.flaky" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| flaky | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="flaky"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.flaky.flaky_probability" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| flaky_probability | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.risk" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| risk | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="risk"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.risk.rank_risk" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| rank_risk | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.model" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| model | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="model"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.rca.model.train" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| train | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#attested-selection" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Attested selection | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Attested selection"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.select.engine" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| engine | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="engine"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.select.engine.select" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| select | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| receipt | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="receipt"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.Receipt" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Receipt | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Receipt"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.Receipt.signed_body" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| signed_body | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.Receipt.verify" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| verify | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.receipt.hash_ids" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| hash_ids | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.keys" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| keys | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="keys"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.keys.resolve_key" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| resolve_key | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.attest.keys.key_id" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| key_id | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#test-generation-self-healing" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Test generation & self-healing | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Test generation & self-healing"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.models" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| models | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="models"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.models.Proposal" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| Proposal | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="Proposal"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.models.Proposal.signed_body" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| signed_body | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.generate" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| generate | |
| </span> | |
| </a> | |
| </li> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.gate" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| gate | |
| </span> | |
| </a> | |
| <nav class="md-nav" aria-label="gate"> | |
| <ul class="md-nav__list"> | |
| <li class="md-nav__item"> | |
| <a href="#assaylab.llm.gate.evaluate_proposal" class="md-nav__link"> | |
| <span class="md-ellipsis"> | |
| evaluate_proposal | |
| </span> | |
| </a> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </li> | |
| </ul> | |
| </nav> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="md-content" data-md-component="content"> | |
| <article class="md-content__inner md-typeset"> | |
| <h1 id="api-reference">API reference<a class="headerlink" href="#api-reference" title="Permanent link">¶</a></h1> | |
| <p>Rendered from source. <code>assaylab</code> keeps a light base wheel; heavy surfaces | |
| (LLM providers, REST) are lazy-imported behind extras.</p> | |
| <h2 id="models">Models<a class="headerlink" href="#models" title="Permanent link">¶</a></h2> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.models" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">models</span> | |
| <a href="#assaylab.models" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>assaylab domain models, built on the agentsensory contract.</p> | |
| <p>The test domain has no pixel (<code>BBox</code>) or time (<code>Span</code>) grounding, so each | |
| issue grounds itself in <code>detail</code> (a JSON-safe dict) carrying the signature id, | |
| affected tests, and file loci. <code>Report</code> / <code>Issue</code> subclass the agentsensory | |
| base types so every result speaks the shared <code>pass/warn/fail</code> + <code>Handoff</code> | |
| language.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.Outcome" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">Outcome</span> | |
| <a href="#assaylab.models.Outcome" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="str">str</span></code>, <code><span title="enum.Enum">Enum</span></code></p> | |
| <p>Normalized outcome of a single test execution.</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.IssueKind" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">IssueKind</span> | |
| <a href="#assaylab.models.IssueKind" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="str">str</span></code>, <code><span title="enum.Enum">Enum</span></code></p> | |
| <p>assaylab issue kinds (the domain vocabulary carried in <code>Issue.kind</code>).</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.IssueSource" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">IssueSource</span> | |
| <a href="#assaylab.models.IssueSource" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="str">str</span></code>, <code><span title="enum.Enum">Enum</span></code></p> | |
| <p>Which analysis produced an issue.</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.TestRecord" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">TestRecord</span> | |
| <a href="#assaylab.models.TestRecord" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="pydantic.BaseModel">BaseModel</span></code></p> | |
| <p>One test execution, normalized across ingestion formats.</p> | |
| <p>This is the universal internal schema every backend maps onto: | |
| <code>(project, commit, test_id, run_id, outcome, duration, message, stacktrace)</code>.</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.FailureSignature" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">FailureSignature</span> | |
| <a href="#assaylab.models.FailureSignature" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="pydantic.BaseModel">BaseModel</span></code></p> | |
| <p>A cluster of failing executions that share a normalized fingerprint.</p> | |
| <p>The fingerprint is a stable hash of the templated (variable-stripped) | |
| error type + message + top stack frames — so runs that differ only in | |
| addresses, line numbers, temp paths or timestamps collapse together.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h5 id="assaylab.models.FailureSignature.to_detail" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">to_detail</span> | |
| <a href="#assaylab.models.FailureSignature.to_detail" class="headerlink" title="Permanent link">¶</a></h5> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">to_detail</span><span class="p">()</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">object</span><span class="p">]</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>JSON-safe grounding payload for an <code>Issue.detail</code>.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.Issue" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">Issue</span> | |
| <a href="#assaylab.models.Issue" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="agentsensory.IssueBase">IssueBase</span></code></p> | |
| <p>assaylab issue — an <code>agentsensory.IssueBase</code> with test-domain grounding in <code>detail</code>.</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.models.Report" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">Report</span> | |
| <a href="#assaylab.models.Report" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="agentsensory.ReportBase">ReportBase</span></code></p> | |
| <p>assaylab report — an <code>agentsensory.ReportBase</code> (verdict + issues + handoff).</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div><h2 id="core-sense">Core sense<a class="headerlink" href="#core-sense" title="Permanent link">¶</a></h2> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.core.analyze" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">analyze</span> | |
| <a href="#assaylab.core.analyze" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Grade test records into an agentsensory :class:<code>Report</code>.</p> | |
| <p>P1 verdict rule (deterministic):</p> | |
| <ul> | |
| <li>no failing records -> PASS</li> | |
| <li>failing records present -> FAIL, one issue per failure signature</li> | |
| <li>a signature absent from a supplied baseline is marked <code>NEW_FAILURE</code> (higher | |
| risk) rather than a plain <code>FAILURE_SIGNATURE</code>.</li> | |
| </ul> | |
| <p>Flaky-vs-real downgrading to WARN arrives in P2; the hook (baseline diff) is | |
| already here.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.core.analyze.grade_records" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">grade_records</span> | |
| <a href="#assaylab.core.analyze.grade_records" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">grade_records</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">records</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">TestRecord</span><span class="p">],</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> <span class="o">*</span><span class="p">,</span> | |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="n">baseline_ids</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="n">backend</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"unknown"</span><span class="p">,</span> | |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Report</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Cluster failures and return a graded :class:<code>Report</code>.</p> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.core.analyze.analyze" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">analyze</span> | |
| <span class="doc doc-labels"> | |
| <small class="doc doc-label doc-label-async"><code>async</code></small> | |
| </span> | |
| <a href="#assaylab.core.analyze.analyze" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">analyze</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">source</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> <span class="o">*</span><span class="p">,</span> | |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="n">backend</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="n">baseline</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> <span class="n">settings</span><span class="p">:</span> <span class="n">Settings</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Report</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Ingest <code>source</code> (and optional <code>baseline</code>) and grade it.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.core.signature" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">signature</span> | |
| <a href="#assaylab.core.signature" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Failure-signature normalization and clustering.</p> | |
| <p>Given failing :class:<code>TestRecord</code> executions, collapse the ones that differ only | |
| in <em>incidental</em> detail — memory addresses, object hashes, line numbers, temp | |
| paths, timestamps, UUIDs, numbers — into a single :class:<code>FailureSignature</code> | |
| identified by a stable fingerprint. This is the evidence layer everything else | |
| (RCA, flaky classification, confidence bounds) grounds on.</p> | |
| <p>Pure-stdlib and deterministic: the same corpus always yields the same | |
| signature ids, so verdicts are reproducible.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.core.signature.normalize" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">normalize</span> | |
| <a href="#assaylab.core.signature.normalize" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">normalize</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Strip incidental variation, returning a stable template string.</p> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.core.signature.fingerprint" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">fingerprint</span> | |
| <a href="#assaylab.core.signature.fingerprint" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">fingerprint</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">record</span><span class="p">:</span> <span class="n">TestRecord</span><span class="p">,</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Return <code>(signature_id, template, exception_type, files)</code> for a failing record.</p> | |
| <p>The template combines the normalized message with normalized top frames, so | |
| two failures with the same root but different incidental text collapse.</p> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.core.signature.cluster" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">cluster</span> | |
| <a href="#assaylab.core.signature.cluster" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">cluster</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">records</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">TestRecord</span><span class="p">],</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">FailureSignature</span><span class="p">]</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Cluster failing records into signatures, ordered by descending run count.</p> | |
| <p>Passing/skipped records are ignored (only failures carry a signature).</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div><h2 id="root-cause-analysis">Root-cause analysis<a class="headerlink" href="#root-cause-analysis" title="Permanent link">¶</a></h2> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.rca.rootcause" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">rootcause</span> | |
| <a href="#assaylab.rca.rootcause" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Root-cause categorization.</p> | |
| <p>A transparent, always-available baseline: map a failure signature to a | |
| root-cause category with a confidence and the evidence that fired. This is the | |
| interpretable floor the learned model (<code>model.py</code>) is measured against, and | |
| what grounds every RCA verdict so a human can audit <em>why</em> a cause was assigned.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.rca.rootcause.categorize" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">categorize</span> | |
| <a href="#assaylab.rca.rootcause.categorize" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">categorize</span><span class="p">(</span><span class="n">sig</span><span class="p">:</span> <span class="n">FailureSignature</span><span class="p">)</span> <span class="o">-></span> <span class="n">Categorization</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Assign a root-cause category to a signature with confidence + evidence.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.rca.flaky" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">flaky</span> | |
| <a href="#assaylab.rca.flaky" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Flaky-vs-real classification.</p> | |
| <p>A failure signature is <em>flaky-leaning</em> when its affected tests behave | |
| non-deterministically — the strongest evidence being a single commit that shows | |
| both a pass and a failure (order-agnostic flakiness, à la iDFlakies), backed by | |
| a high flip-rate across runs. Real failures are consistent at a given commit.</p> | |
| <p>The heuristic here is always available; :func:<code>flaky_probability</code> optionally | |
| defers to a learned model when one is supplied.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.rca.flaky.flaky_probability" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">flaky_probability</span> | |
| <a href="#assaylab.rca.flaky.flaky_probability" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">flaky_probability</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">features</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">],</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> <span class="n">tests</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> | |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="n">stats</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">TestStats</span><span class="p">],</span> | |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="n">model</span><span class="p">:</span> <span class="nb">object</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="p">)</span> <span class="o">-></span> <span class="n">FlakyVerdict</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Learned probability when <code>model</code> is given, else the heuristic.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.rca.risk" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">risk</span> | |
| <a href="#assaylab.rca.risk" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Per-test risk scoring and next-run failure forecasting.</p> | |
| <p>Risk blends recency-weighted failure rate with instability (flip-rate): a test | |
| that fails often <em>and</em> recently is high risk; a rarely-failing stable test is | |
| low risk. <code>forecast</code> is the model's estimate that the test fails on its next | |
| run, used later (P3) to bound the confidence lost when a test is skipped.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.rca.risk.rank_risk" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">rank_risk</span> | |
| <a href="#assaylab.rca.risk.rank_risk" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">rank_risk</span><span class="p">(</span><span class="n">stats</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">TestStats</span><span class="p">])</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">Risk</span><span class="p">]</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>All tests ranked by descending risk.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.rca.model" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">model</span> | |
| <a href="#assaylab.rca.model" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>A small, self-contained logistic-regression model for flaky prediction.</p> | |
| <p>Pure Python (batch gradient descent) — no numpy/scikit-learn in the base wheel, | |
| and the trained model serializes to <strong>JSON</strong>, never pickle. That matters: a | |
| pickled model is arbitrary-code-execution on load, so a model pulled from a | |
| registry or CI cache would be a supply-chain hole. JSON weights load inertly.</p> | |
| <p>Features are addressed by name (a dict), so callers never depend on column | |
| order. Unknown/missing features default to 0.0. Inputs are standardized using | |
| means/stds captured at fit time.</p> | |
| <p>For heavier models, train externally and export coefficients into this JSON | |
| shape; prediction stays dependency-free.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.rca.model.train" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">train</span> | |
| <a href="#assaylab.rca.model.train" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">train</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">samples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">],</span> <span class="nb">int</span><span class="p">]],</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> <span class="o">*</span><span class="p">,</span> | |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="n">feature_names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="n">epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">400</span><span class="p">,</span> | |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> <span class="n">lr</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.3</span><span class="p">,</span> | |
| <a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a> <span class="n">l2</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.001</span><span class="p">,</span> | |
| <a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="p">)</span> <span class="o">-></span> <span class="n">LogisticModel</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Fit a logistic model on <code>(features, label)</code> pairs (label in {0,1}).</p> | |
| <p>Deterministic: no randomness, weights start at zero, so the same data always | |
| yields the same model.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div><h2 id="attested-selection">Attested selection<a class="headerlink" href="#attested-selection" title="Permanent link">¶</a></h2> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.select.engine" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">engine</span> | |
| <a href="#assaylab.select.engine" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Test-selection engine with a verifiable confidence bound.</p> | |
| <p>Each candidate test <code>t</code> has a detection probability <code>q_t</code> — the modelled | |
| chance it fails (catches a regression) on this run, estimated from history | |
| (P2 forecast). If we run a subset <code>S</code> and skip <code>U = All \ S</code>, the | |
| probability that <em>at least one</em> skipped test would have failed is</p> | |
| <div class="highlight"><pre><span></span><code>epsilon = 1 - prod_{t in U} (1 - q_t) | |
| </code></pre></div> | |
| <p>under an independence assumption. <code>epsilon</code> is the <strong>confidence lost</strong> by | |
| skipping — an upper-bound proxy for the chance we miss a regression the suite | |
| would have caught. Selection keeps the highest value-density tests | |
| (<code>q_t</code> per second) until either <code>epsilon <= target_epsilon</code> or the | |
| <code>time_budget_s</code> is exhausted; the achieved <code>epsilon</code> is always reported.</p> | |
| <p>Honest limits (stated, not hidden): independence of failures, <code>q_t</code> | |
| stationarity, and coverage only of regression classes seen historically. These | |
| are the receipt's residual assumptions, not guarantees.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.select.engine.select" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">select</span> | |
| <a href="#assaylab.select.engine.select" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">select</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">candidates</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">Candidate</span><span class="p">],</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> <span class="o">*</span><span class="p">,</span> | |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="n">target_epsilon</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="n">time_budget_s</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Selection</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Choose a subset to run under a confidence target and/or a time budget.</p> | |
| <p>Exactly one of <code>target_epsilon</code> / <code>time_budget_s</code> should be the primary | |
| driver; if both are given, selection stops as soon as <em>either</em> is satisfied | |
| (target met) or violated (budget spent).</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.attest.receipt" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">receipt</span> | |
| <a href="#assaylab.attest.receipt" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>The attested receipt: a signed commitment to a test-selection outcome.</p> | |
| <p>The signature (HMAC-SHA256) covers the <em>result</em> — the inputs hash, the selected | |
| and skipped sets (by hash + count), and the computed confidence bound <code>epsilon</code> | |
| — so it binds what actually happened, not merely that a run occurred. Verifying | |
| is constant-time and, given the candidate inputs, re-derivable: a consumer can | |
| recompute <code>epsilon</code> from the committed inputs and check it matches.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.attest.receipt.Receipt" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">Receipt</span> | |
| <a href="#assaylab.attest.receipt.Receipt" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="pydantic.BaseModel">BaseModel</span></code></p> | |
| <p>A signed test-selection receipt. <code>signature</code> is not part of the signed body.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h5 id="assaylab.attest.receipt.Receipt.signed_body" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">signed_body</span> | |
| <a href="#assaylab.attest.receipt.Receipt.signed_body" class="headerlink" title="Permanent link">¶</a></h5> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">signed_body</span><span class="p">()</span> <span class="o">-></span> <span class="nb">bytes</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Canonical bytes covered by the signature (everything but <code>signature</code>).</p> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-function"> | |
| <h5 id="assaylab.attest.receipt.Receipt.verify" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">verify</span> | |
| <a href="#assaylab.attest.receipt.Receipt.verify" class="headerlink" title="Permanent link">¶</a></h5> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">verify</span><span class="p">(</span><span class="n">key</span><span class="p">:</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Constant-time signature check.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.attest.receipt.hash_ids" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">hash_ids</span> | |
| <a href="#assaylab.attest.receipt.hash_ids" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">hash_ids</span><span class="p">(</span><span class="n">ids</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="nb">str</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Stable hash of a set of test ids (order-independent).</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.attest.keys" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">keys</span> | |
| <a href="#assaylab.attest.keys" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Signing-key resolution — env, else a persisted per-installation random key.</p> | |
| <p>Never a hardcoded/default secret (a shipped default key lets anyone forge a | |
| receipt). Resolution order:</p> | |
| <ol> | |
| <li><code>ASSAYLAB_SIGNING_KEY</code> — an <strong>explicitly encoded</strong> key: <code>hex:<hex></code>, | |
| <code>base64:<b64></code>, or <code>raw:<utf8></code> (or a bare value, treated as raw). The | |
| encoding is never guessed, so one string can't resolve to two different keys. | |
| Must carry >= 16 bytes and not be a degenerate (single-byte-repeated) value.</li> | |
| <li>A per-installation key persisted at <code><user_config>/assaylab/signing.key</code>, | |
| created with :func:<code>secrets.token_bytes</code> and <code>0600</code> perms on first use.</li> | |
| </ol> | |
| <p>Both paths <strong>refuse to follow symlinks and reject files not owned by the | |
| current user</strong>, closing the symlink/TOCTOU forgery vector. A malformed env key | |
| fails closed (raises) rather than silently falling back.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.attest.keys.resolve_key" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">resolve_key</span> | |
| <a href="#assaylab.attest.keys.resolve_key" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">resolve_key</span><span class="p">()</span> <span class="o">-></span> <span class="nb">bytes</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Return the signing key (env override, else persisted per-installation key).</p> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.attest.keys.key_id" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">key_id</span> | |
| <a href="#assaylab.attest.keys.key_id" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">key_id</span><span class="p">(</span><span class="n">key</span><span class="p">:</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>A short, non-secret identifier for a key (first bytes of its SHA-256).</p> | |
| <p>Lets a receipt name <em>which</em> key signed it without revealing the key.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div><h2 id="test-generation-self-healing">Test generation & self-healing<a class="headerlink" href="#test-generation-self-healing" title="Permanent link">¶</a></h2> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.llm.models" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">models</span> | |
| <a href="#assaylab.llm.models" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>The Proposal: an LLM artifact assaylab never executes or applies.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-class"> | |
| <h4 id="assaylab.llm.models.Proposal" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-class-name">Proposal</span> | |
| <a href="#assaylab.llm.models.Proposal" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc doc-contents "> | |
| <p class="doc doc-class-bases"> | |
| Bases: <code><span title="pydantic.BaseModel">BaseModel</span></code></p> | |
| <p>A dry-run suggestion + its provenance and acceptance criterion.</p> | |
| <p><code>applied</code> is always False: assaylab produces the proposal and defines how | |
| to verify it, but a human/CI applies and runs it in their own sandbox. The | |
| verdict layer decides acceptance (see :mod:<code>assaylab.llm.gate</code>).</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h5 id="assaylab.llm.models.Proposal.signed_body" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">signed_body</span> | |
| <a href="#assaylab.llm.models.Proposal.signed_body" class="headerlink" title="Permanent link">¶</a></h5> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">signed_body</span><span class="p">()</span> <span class="o">-></span> <span class="nb">bytes</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Canonical bytes covered by the signature (everything but the signature).</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.llm.generate" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">generate</span> | |
| <a href="#assaylab.llm.generate" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Propose a regression test from a failure signature (dry-run).</p> | |
| <div class="doc doc-children"> | |
| </div> | |
| </div> | |
| </div> | |
| <div class="doc doc-object doc-module"> | |
| <h3 id="assaylab.llm.gate" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-module-name">gate</span> | |
| <a href="#assaylab.llm.gate" class="headerlink" title="Permanent link">¶</a></h3> | |
| <div class="doc doc-contents first"> | |
| <p>Gate a proposal against a REAL run — acceptance flows through the verdict layer.</p> | |
| <p>assaylab never runs the proposed code. Instead the user applies + runs it in | |
| their own sandbox and feeds the resulting test output back here; the proposal is | |
| accepted only if that graded run meets its acceptance criterion. This is what | |
| keeps LLM output honest: a generated test counts only if it actually reproduces | |
| the bug; a heal counts only if the flaky signature stops failing.</p> | |
| <div class="doc doc-children"> | |
| <div class="doc doc-object doc-function"> | |
| <h4 id="assaylab.llm.gate.evaluate_proposal" class="doc doc-heading"> | |
| <span class="doc doc-object-name doc-function-name">evaluate_proposal</span> | |
| <a href="#assaylab.llm.gate.evaluate_proposal" class="headerlink" title="Permanent link">¶</a></h4> | |
| <div class="doc-signature highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="nf">evaluate_proposal</span><span class="p">(</span> | |
| <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> <span class="n">proposal</span><span class="p">:</span> <span class="n">Proposal</span><span class="p">,</span> | |
| <a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> <span class="n">result_source</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> | |
| <a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="o">*</span><span class="p">,</span> | |
| <a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="n">backend</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> <span class="n">settings</span><span class="p">:</span> <span class="n">Settings</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> | |
| <a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Evaluation</span> | |
| </code></pre></div> | |
| <div class="doc doc-contents "> | |
| <p>Grade the returned run and decide whether the proposal is accepted.</p> | |
| <p>The proposal is treated as UNTRUSTED: its signature must verify (it was | |
| signed at generation by the tool that derived the criteria from a real | |
| signature), or we refuse to grade it. This stops a hand-crafted proposal | |
| JSON from weakening its own acceptance criteria.</p> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </div> | |
| </article> | |
| </div> | |
| <script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script> | |
| </div> | |
| <button type="button" class="md-top md-icon" data-md-component="top" hidden> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg> | |
| Back to top | |
| </button> | |
| </main> | |
| <footer class="md-footer"> | |
| <nav class="md-footer__inner md-grid" aria-label="Footer" > | |
| <a href="../evaluation/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Evaluation"> | |
| <div class="md-footer__button md-icon"> | |
| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg> | |
| </div> | |
| <div class="md-footer__title"> | |
| <span class="md-footer__direction"> | |
| Previous | |
| </span> | |
| <div class="md-ellipsis"> | |
| Evaluation | |
| </div> | |
| </div> | |
| </a> | |
| </nav> | |
| <div class="md-footer-meta md-typeset"> | |
| <div class="md-footer-meta__inner md-grid"> | |
| <div class="md-copyright"> | |
| <div class="md-copyright__highlight"> | |
| MIT © Amit Patole | |
| </div> | |
| Made with | |
| <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener"> | |
| Material for MkDocs | |
| </a> | |
| </div> | |
| </div> | |
| </div> | |
| </footer> | |
| </div> | |
| <div class="md-dialog" data-md-component="dialog"> | |
| <div class="md-dialog__inner md-typeset"></div> | |
| </div> | |
| <script id="__config" type="application/json">{"annotate": null, "base": "..", "features": ["navigation.sections", "navigation.top", "navigation.tracking", "navigation.instant", "navigation.footer", "content.code.copy", "content.code.annotate", "toc.follow", "search.suggest", "search.highlight"], "search": "../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script> | |
| <script src="../assets/javascripts/bundle.79ae519e.min.js"></script> | |
| </body> | |
| </html> |