{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "machine_shape": "hm", "gpuType": "G4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "83d1d83f941942e5877d10dda18831ef": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5484cc8c987c4b90972056a2a1ad647e", "IPY_MODEL_eeaa111590424f478fb4d7372d0949d5", "IPY_MODEL_400aabdf6f71482caf36df6cb7650c5e" ], "layout": "IPY_MODEL_b912b58d41b34c35b545119b1124e1a9" } }, "5484cc8c987c4b90972056a2a1ad647e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_371c6596a5ba426c944ee344bc82e15a", "placeholder": "​", "style": "IPY_MODEL_5cd57eea1d3649d9a929a233ef9cf1b3", "value": "README.md: 100%" } }, "eeaa111590424f478fb4d7372d0949d5": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d44f66cc06f34380979de8619df5052d", "max": 10464, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_bb2b5d1d2f3945e1b03a8845df7600d9", "value": 10464 } }, "400aabdf6f71482caf36df6cb7650c5e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4445cf712c5648a291608f0d28b2c546", "placeholder": "​", "style": "IPY_MODEL_4053dbd62e1a4997bb211d8a061ffabf", "value": " 10.5k/10.5k [00:00<00:00, 2.98MB/s]" } }, "b912b58d41b34c35b545119b1124e1a9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "371c6596a5ba426c944ee344bc82e15a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5cd57eea1d3649d9a929a233ef9cf1b3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d44f66cc06f34380979de8619df5052d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "bb2b5d1d2f3945e1b03a8845df7600d9": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "4445cf712c5648a291608f0d28b2c546": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4053dbd62e1a4997bb211d8a061ffabf": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "50e150adf12c4eac83c5bbbcba495436": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f82a7651dcfd4912bc86603194e194cf", "IPY_MODEL_f38eb0917dfe4fc2bc26a687b4d79c21", "IPY_MODEL_2708c3a73e7145c8955146cd5547a91e" ], "layout": "IPY_MODEL_7121279632654d52adcc7eff3aace621" } }, "f82a7651dcfd4912bc86603194e194cf": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_dffad1378e614dc2921cb50326fe3170", "placeholder": "​", "style": "IPY_MODEL_887bf4165de24db8aecf648ff3ff0fb5", "value": "wikitext-103-raw-v1/test-00000-of-00001.(…): 100%" } }, "f38eb0917dfe4fc2bc26a687b4d79c21": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_60712f450afb443683564734a6013d0b", "max": 732610, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_8fe736e3baa2475189bc9f9a79dcd5d0", "value": 732610 } }, "2708c3a73e7145c8955146cd5547a91e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_798d0ff4fcb74bd6842425a14d865bb2", "placeholder": "​", "style": "IPY_MODEL_fc31527a15c049ea968b7a0727329bcb", "value": " 733k/733k [00:00<00:00, 942kB/s]" } }, "7121279632654d52adcc7eff3aace621": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "dffad1378e614dc2921cb50326fe3170": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "887bf4165de24db8aecf648ff3ff0fb5": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "60712f450afb443683564734a6013d0b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8fe736e3baa2475189bc9f9a79dcd5d0": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "798d0ff4fcb74bd6842425a14d865bb2": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "fc31527a15c049ea968b7a0727329bcb": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "37238e991469495caee69628b6ea1956": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f91c35e715c7440d9db9cf88ff834a87", "IPY_MODEL_dcf30fd8ca1743c799fedde260d4770e", "IPY_MODEL_1ccc2d2dd89845d4a9f94da3f27e85cd" ], "layout": "IPY_MODEL_9ee3cbae883a4743b6d1610b4d0a53de" } }, "f91c35e715c7440d9db9cf88ff834a87": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4c6ad259d67049a085d256445911e6cf", "placeholder": "​", "style": "IPY_MODEL_89b2237ac6fe4e44bd48c198c9d53c48", "value": "wikitext-103-raw-v1/train-00000-of-00002(…): 100%" } }, "dcf30fd8ca1743c799fedde260d4770e": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7a6e63d395e94e5a99edd973db882b5e", "max": 156987808, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_ea161dc0e5ae400f85c28dc8f5fd276f", "value": 156987808 } }, "1ccc2d2dd89845d4a9f94da3f27e85cd": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_76b0687fe246471d9008590a9bad7153", "placeholder": "​", "style": "IPY_MODEL_5575dd05ed4e4c2daeb1d0d96742a499", "value": " 157M/157M [00:01<00:00, 743MB/s]" } }, "9ee3cbae883a4743b6d1610b4d0a53de": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4c6ad259d67049a085d256445911e6cf": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "89b2237ac6fe4e44bd48c198c9d53c48": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7a6e63d395e94e5a99edd973db882b5e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ea161dc0e5ae400f85c28dc8f5fd276f": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "76b0687fe246471d9008590a9bad7153": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5575dd05ed4e4c2daeb1d0d96742a499": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "12dd0d2094df41389d65356f0237e67a": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5efd12e892ae47debb1fbf6ee81c0251", "IPY_MODEL_bf95ac59c2cc489c8f58afeef48198ca", "IPY_MODEL_7226093a972c43129145f336ac7c87d9" ], "layout": "IPY_MODEL_269f7e0e536e4d73917a47aa8af29400" } }, "5efd12e892ae47debb1fbf6ee81c0251": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4500d3252dd140cb915667797ab1e440", "placeholder": "​", "style": "IPY_MODEL_fffb559b9ce24b97bb4002c381ba3f51", "value": "wikitext-103-raw-v1/train-00001-of-00002(…): 100%" } }, "bf95ac59c2cc489c8f58afeef48198ca": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e3b4475f645d4ca189ad0655cbaac298", "max": 157088770, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_e84425f852614e4993dcec4f74a7ea46", "value": 157088770 } }, "7226093a972c43129145f336ac7c87d9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7146d0488639440b80ac105efb0bcf11", "placeholder": "​", "style": "IPY_MODEL_0609e309f35c459cb46f432f39ce5b39", "value": " 157M/157M [00:01<00:00, 118MB/s]" } }, "269f7e0e536e4d73917a47aa8af29400": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4500d3252dd140cb915667797ab1e440": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "fffb559b9ce24b97bb4002c381ba3f51": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e3b4475f645d4ca189ad0655cbaac298": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e84425f852614e4993dcec4f74a7ea46": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "7146d0488639440b80ac105efb0bcf11": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0609e309f35c459cb46f432f39ce5b39": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "9893c161f37f4409b9267abe5c80afce": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_111f5d3ceca84420a33e0d6811d61be0", "IPY_MODEL_4992d11daab6406e8ee6aa36014e93c5", "IPY_MODEL_9e520a8a2af64f2e8efc02d0507eca04" ], "layout": "IPY_MODEL_c479d075853b4463bc3dfbe85b432ab6" } }, "111f5d3ceca84420a33e0d6811d61be0": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4d0a0a58d7f5437fb044d45cd7a55d23", "placeholder": "​", "style": "IPY_MODEL_86e6fdce583242f1a24f386cfc276ff1", "value": "wikitext-103-raw-v1/validation-00000-of-(…): 100%" } }, "4992d11daab6406e8ee6aa36014e93c5": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_caed3676a4744808af3d3b899206c23c", "max": 657209, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_b3a8fab92e41487d980d899ab73515d8", "value": 657209 } }, "9e520a8a2af64f2e8efc02d0507eca04": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8c460ad9d9754f0eac89d4c98bd03350", "placeholder": "​", "style": "IPY_MODEL_662a8c74eccc4780801b356887b19ad3", "value": " 657k/657k [00:00<00:00, 1.05MB/s]" } }, "c479d075853b4463bc3dfbe85b432ab6": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4d0a0a58d7f5437fb044d45cd7a55d23": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "86e6fdce583242f1a24f386cfc276ff1": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "caed3676a4744808af3d3b899206c23c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b3a8fab92e41487d980d899ab73515d8": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "8c460ad9d9754f0eac89d4c98bd03350": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "662a8c74eccc4780801b356887b19ad3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "521bbe2d2c5940e8839ea7a649f23164": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_047dc9fe110c4bc18641fc6366b2e0de", "IPY_MODEL_ebf222094ac14b3392c7b5dfe199ddae", "IPY_MODEL_9d55ee1c086f426eb573b0eb0886bcb5" ], "layout": "IPY_MODEL_2ef6b544d49c482a85cb59fee51aaa41" } }, "047dc9fe110c4bc18641fc6366b2e0de": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_03b16f9eb06d44e898b84bce8da58ced", "placeholder": "​", "style": "IPY_MODEL_1546b0d321eb416183f14b94785b48a8", "value": "Generating test split: 100%" } }, "ebf222094ac14b3392c7b5dfe199ddae": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_05fc7e21c5cb42c7b7738e41bb822612", "max": 4358, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_8c2dae19f12e4e99908980e41ddaadc2", "value": 4358 } }, "9d55ee1c086f426eb573b0eb0886bcb5": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f27db7cc8e3145708c84d0527de74c60", "placeholder": "​", "style": "IPY_MODEL_9e3ecb45249b4367a98c60bad71d2873", "value": " 4358/4358 [00:00<00:00, 158591.47 examples/s]" } }, "2ef6b544d49c482a85cb59fee51aaa41": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "03b16f9eb06d44e898b84bce8da58ced": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1546b0d321eb416183f14b94785b48a8": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "05fc7e21c5cb42c7b7738e41bb822612": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8c2dae19f12e4e99908980e41ddaadc2": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "f27db7cc8e3145708c84d0527de74c60": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9e3ecb45249b4367a98c60bad71d2873": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "83a79438586c45caafca24b7f3dc2449": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a5d6a4a8ee944d2c83df5dd2a89c33d9", "IPY_MODEL_6c97d3e49fab4f9f8015537705e92ddf", "IPY_MODEL_0ffef704b9a64e1189c44145ddc6eeb7" ], "layout": "IPY_MODEL_cae54e625ff241f687fba6d3cb7ab241" } }, "a5d6a4a8ee944d2c83df5dd2a89c33d9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e9d638d1dc69450e97e8d875694acfc1", "placeholder": "​", "style": "IPY_MODEL_400cd959440f496fa2b3295d6ddfed75", "value": "Generating train split: 100%" } }, "6c97d3e49fab4f9f8015537705e92ddf": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d524ae6998a04a9a9adaede5917d1b2f", "max": 1801350, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_8a2a22afa3184a88b3f6ee57bd0811d1", "value": 1801350 } }, "0ffef704b9a64e1189c44145ddc6eeb7": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d59c2f1df60543b3a081bc8bfd6f1f2f", "placeholder": "​", "style": "IPY_MODEL_8cbaa69fb86d489ba01f305993160710", "value": " 1801350/1801350 [00:00<00:00, 2077562.87 examples/s]" } }, "cae54e625ff241f687fba6d3cb7ab241": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e9d638d1dc69450e97e8d875694acfc1": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "400cd959440f496fa2b3295d6ddfed75": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d524ae6998a04a9a9adaede5917d1b2f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8a2a22afa3184a88b3f6ee57bd0811d1": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "d59c2f1df60543b3a081bc8bfd6f1f2f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8cbaa69fb86d489ba01f305993160710": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "55c498bc304947c2805cf7db7e6cfd95": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5f419d0cc5aa49caae4aec2e3257a5db", "IPY_MODEL_e0ab10aa67114492a2f77587b1570783", "IPY_MODEL_ebf6e51dc43b439a8842b9957d7db8eb" ], "layout": "IPY_MODEL_8d4b5312ea304c0ea732ed1b096610da" } }, "5f419d0cc5aa49caae4aec2e3257a5db": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ea40ffbd34c743369f31595038d22c98", "placeholder": "​", "style": "IPY_MODEL_5c1698382dee4cb79498be10ca625193", "value": "Generating validation split: 100%" } }, "e0ab10aa67114492a2f77587b1570783": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_87c0bd1330e84a8d88d0a2c03619e9fa", "max": 3760, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_e3e1258c184a4cdb9f17cae93458e070", "value": 3760 } }, "ebf6e51dc43b439a8842b9957d7db8eb": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8155a51d8a5f4b7b9872bf5a0de61407", "placeholder": "​", "style": "IPY_MODEL_81e301c3f8804a31b880c8ad8a6ddc55", "value": " 3760/3760 [00:00<00:00, 711412.08 examples/s]" } }, "8d4b5312ea304c0ea732ed1b096610da": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ea40ffbd34c743369f31595038d22c98": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5c1698382dee4cb79498be10ca625193": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "87c0bd1330e84a8d88d0a2c03619e9fa": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e3e1258c184a4cdb9f17cae93458e070": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "8155a51d8a5f4b7b9872bf5a0de61407": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "81e301c3f8804a31b880c8ad8a6ddc55": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6733b18541f647e286c01d09483912e1": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_bf7d9ebb6f8f42bab6d2bdd8c54b0af0", "IPY_MODEL_b0da5e31e49849b082a9dd713b35a84f", "IPY_MODEL_7007d889a446427b90e73f7d55f2823d" ], "layout": "IPY_MODEL_985c132d01f34f12aa6fdbfefffe76a8" } }, "bf7d9ebb6f8f42bab6d2bdd8c54b0af0": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f9d8f8d0b0894a2691fbfdb1986e83d7", "placeholder": "​", "style": "IPY_MODEL_6a4ee6077e3843b8aa0a422a808be524", "value": "README.md: 100%" } }, "b0da5e31e49849b082a9dd713b35a84f": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4fe3c25911d4436698a113969999257e", "max": 10464, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_fa67416dee644db289200d5cfb3b244a", "value": 10464 } }, "7007d889a446427b90e73f7d55f2823d": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ce48f8202ddf47cd9dc682f50cfd2f11", "placeholder": "​", "style": "IPY_MODEL_a241b526154e41e19cf7f86dc5d413c8", "value": " 10.5k/10.5k [00:00<00:00, 3.11MB/s]" } }, "985c132d01f34f12aa6fdbfefffe76a8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f9d8f8d0b0894a2691fbfdb1986e83d7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6a4ee6077e3843b8aa0a422a808be524": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "4fe3c25911d4436698a113969999257e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "fa67416dee644db289200d5cfb3b244a": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "ce48f8202ddf47cd9dc682f50cfd2f11": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a241b526154e41e19cf7f86dc5d413c8": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5544fffa9f774b84b55e58e7dc67aed7": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_bd3248211f7b4a25aa6bc8ca5d585af3", "IPY_MODEL_b62582c307234c9d91db9fee83c23b62", "IPY_MODEL_d823562bae52442a9539460c934f71d2" ], "layout": "IPY_MODEL_e35a04a752a44c0392c29a863cbaed20" } }, "bd3248211f7b4a25aa6bc8ca5d585af3": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_44c143c321414c33ae99f753d1354c7d", "placeholder": "​", "style": "IPY_MODEL_01caddb991c64455b64b1e33b0c87b41", "value": "wikitext-103-raw-v1/test-00000-of-00001.(…): 100%" } }, "b62582c307234c9d91db9fee83c23b62": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_97a1dc073fc846d1a97a5abaeeb5c616", "max": 732610, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_9437f3e441054770921d99c587237508", "value": 732610 } }, "d823562bae52442a9539460c934f71d2": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_697e26c0a0364989ac983ec5e785f2bd", "placeholder": "​", "style": "IPY_MODEL_abc2f44a38f649d4a7d97144306644f3", "value": " 733k/733k [00:00<00:00, 1.39MB/s]" } }, "e35a04a752a44c0392c29a863cbaed20": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "44c143c321414c33ae99f753d1354c7d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "01caddb991c64455b64b1e33b0c87b41": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "97a1dc073fc846d1a97a5abaeeb5c616": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9437f3e441054770921d99c587237508": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "697e26c0a0364989ac983ec5e785f2bd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "abc2f44a38f649d4a7d97144306644f3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e9726a9df5154b8180bdb709ada22c5c": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f92792302ba1456591c24ac7e4ecd89a", "IPY_MODEL_5d57c42ac7a2491b876567559abdc1e3", "IPY_MODEL_d0be94efaf5b4f3fbdaf1df5748dd03f" ], "layout": "IPY_MODEL_4c65959847034e55868634793d494c12" } }, "f92792302ba1456591c24ac7e4ecd89a": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_83814b55220245a3971a5f042e1358f9", "placeholder": "​", "style": "IPY_MODEL_757c3bc75b4f43a6b8eb407d27c3959b", "value": "wikitext-103-raw-v1/train-00000-of-00002(…): 100%" } }, "5d57c42ac7a2491b876567559abdc1e3": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_28d1f1856bc245aaa3f67e2df10ae9ba", "max": 156987808, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_209da5ac5a414b47915ed69b683418f4", "value": 156987808 } }, "d0be94efaf5b4f3fbdaf1df5748dd03f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8d2718f3563044f9aaef968a8a611efd", "placeholder": "​", "style": "IPY_MODEL_d03a6a12939047799acd1e2e0f39a268", "value": " 157M/157M [00:01<00:00, 171MB/s]" } }, "4c65959847034e55868634793d494c12": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "83814b55220245a3971a5f042e1358f9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "757c3bc75b4f43a6b8eb407d27c3959b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "28d1f1856bc245aaa3f67e2df10ae9ba": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "209da5ac5a414b47915ed69b683418f4": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "8d2718f3563044f9aaef968a8a611efd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d03a6a12939047799acd1e2e0f39a268": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "a9fb8e7ea07e4a1f9e6973015af74b7f": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_3f581a8eded546e6a2a4196bff5e4629", "IPY_MODEL_2517b9b41f1440dba98d167fdd57064b", "IPY_MODEL_8709459dde304673ac0ac86d0921ba9c" ], "layout": "IPY_MODEL_f27d39cfc676444fb756875d3843d1e6" } }, "3f581a8eded546e6a2a4196bff5e4629": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3c65c222c3e449d8853b9d0b89a4fa0d", "placeholder": "​", "style": "IPY_MODEL_ab30536956cd402a9b4ce26818a0f737", "value": "wikitext-103-raw-v1/train-00001-of-00002(…): 100%" } }, "2517b9b41f1440dba98d167fdd57064b": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cb6af780886d44cebb925b6ab9fdfa40", "max": 157088770, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_d445523d9bcb47d1bdbf105bdaa6992a", "value": 157088770 } }, "8709459dde304673ac0ac86d0921ba9c": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bf5b6b452ede4ddd846690f6baeea4f6", "placeholder": "​", "style": "IPY_MODEL_32dc5aad1ffd4ac0a24d4ae2d4ad7253", "value": " 157M/157M [00:01<00:00, 1.56GB/s]" } }, "f27d39cfc676444fb756875d3843d1e6": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3c65c222c3e449d8853b9d0b89a4fa0d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ab30536956cd402a9b4ce26818a0f737": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cb6af780886d44cebb925b6ab9fdfa40": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d445523d9bcb47d1bdbf105bdaa6992a": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "bf5b6b452ede4ddd846690f6baeea4f6": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "32dc5aad1ffd4ac0a24d4ae2d4ad7253": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "9746d2db002e4268abd96ca59377d602": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e75d909801db46cd994f6901b0c88f18", "IPY_MODEL_1a709a29aba745e2b61f6dfa3ce08789", "IPY_MODEL_23248de38bd14e7f9b0600452623f67e" ], "layout": "IPY_MODEL_4a5c5b86e3534ca2928a64a8c47112fc" } }, "e75d909801db46cd994f6901b0c88f18": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f79ec18de4d24bb2b808324977cdcd16", "placeholder": "​", "style": "IPY_MODEL_7425b0cea632486cbcfa0fbebc55f18b", "value": "wikitext-103-raw-v1/validation-00000-of-(…): 100%" } }, "1a709a29aba745e2b61f6dfa3ce08789": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a2e42797a81244069b8f223645bcaf8c", "max": 657209, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_e10272655204493eba8ebab078d2c6c7", "value": 657209 } }, "23248de38bd14e7f9b0600452623f67e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_036a1029e0574a959c6c1176f15351e4", "placeholder": "​", "style": "IPY_MODEL_8e7a756a4645493e8ac4fdd7d017f1a4", "value": " 657k/657k [00:00<00:00, 1.05MB/s]" } }, "4a5c5b86e3534ca2928a64a8c47112fc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f79ec18de4d24bb2b808324977cdcd16": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7425b0cea632486cbcfa0fbebc55f18b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "a2e42797a81244069b8f223645bcaf8c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e10272655204493eba8ebab078d2c6c7": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "036a1029e0574a959c6c1176f15351e4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8e7a756a4645493e8ac4fdd7d017f1a4": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6f71c712ab2b419fb9e09727f1bae912": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_97792cf5f93640628705b3f50c66f25d", "IPY_MODEL_458bb0d5bc254ab6ad2acf061ee9e920", "IPY_MODEL_9a14b826555047a297591de433ffce1c" ], "layout": "IPY_MODEL_80740ea00d8d42c594b08baf11a8f16a" } }, "97792cf5f93640628705b3f50c66f25d": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_402d6337f56e4943875d8bfe67516b51", "placeholder": "​", "style": "IPY_MODEL_ad60e58e35d84e509a75246f57e9e921", "value": "Generating test split: 100%" } }, "458bb0d5bc254ab6ad2acf061ee9e920": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7bd66aea01e8413490c9d25845c451a3", "max": 4358, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_99d4f38b254b481cb2e554852a5c9b02", "value": 4358 } }, "9a14b826555047a297591de433ffce1c": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e6b7ae5466c44b1ea2ef94ef0b34988b", "placeholder": "​", "style": "IPY_MODEL_31491fe8fa8a4cc8ae10ba7dc34c02f4", "value": " 4358/4358 [00:00<00:00, 155774.85 examples/s]" } }, "80740ea00d8d42c594b08baf11a8f16a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "402d6337f56e4943875d8bfe67516b51": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ad60e58e35d84e509a75246f57e9e921": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7bd66aea01e8413490c9d25845c451a3": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "99d4f38b254b481cb2e554852a5c9b02": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "e6b7ae5466c44b1ea2ef94ef0b34988b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "31491fe8fa8a4cc8ae10ba7dc34c02f4": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "85c335c0e3634b90aab122c9dbfe8bdf": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_98a113812d1f40c5b4a7df00f437fc57", "IPY_MODEL_104c0330a7924d6c968bab19d47d821c", "IPY_MODEL_8f33187de10749898eed245e3f29bd2f" ], "layout": "IPY_MODEL_cf1bc2a8be6a4d1d9dbc932ec7ccb74f" } }, "98a113812d1f40c5b4a7df00f437fc57": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_46af79a6fcae48728b6aa74b1bf22f3d", "placeholder": "​", "style": "IPY_MODEL_c9bad91fe7e1472b8423f1de82c87ef0", "value": "Generating train split: 100%" } }, "104c0330a7924d6c968bab19d47d821c": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e3c97785318b4fd4858afe7749a28974", "max": 1801350, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_f402b8cf12454f6984d2a98fb6161efa", "value": 1801350 } }, "8f33187de10749898eed245e3f29bd2f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ad6375b73c554213975e80528318bfa3", "placeholder": "​", "style": "IPY_MODEL_507a01d7449041568ca10f94735eae58", "value": " 1801350/1801350 [00:01<00:00, 1931057.75 examples/s]" } }, "cf1bc2a8be6a4d1d9dbc932ec7ccb74f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "46af79a6fcae48728b6aa74b1bf22f3d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c9bad91fe7e1472b8423f1de82c87ef0": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e3c97785318b4fd4858afe7749a28974": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f402b8cf12454f6984d2a98fb6161efa": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "ad6375b73c554213975e80528318bfa3": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "507a01d7449041568ca10f94735eae58": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "c35f05059a7d423f9758265cfc2d45e5": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_8ef2e29e4e3746bd858d377837659065", "IPY_MODEL_a0303f4db2a745e3bd86b9437cf3b4f0", "IPY_MODEL_927db1fe63c6410294c4cd3eabab5e68" ], "layout": "IPY_MODEL_c4c50f17cd50456e8749134980953dcd" } }, "8ef2e29e4e3746bd858d377837659065": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f18ce9c9c3614872b9c6f58b60320e17", "placeholder": "​", "style": "IPY_MODEL_67d1880b56f4410fa1107c05f30c403c", "value": "Generating validation split: 100%" } }, "a0303f4db2a745e3bd86b9437cf3b4f0": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8a0b7ed6756047aa99ff1a3742071450", "max": 3760, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_6fd164f995d943d1a50a3e41f134fdc2", "value": 3760 } }, "927db1fe63c6410294c4cd3eabab5e68": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4c8cdc6dce984019ac5a13e89f9c4f1b", "placeholder": "​", "style": "IPY_MODEL_033bf4e55a0f46f59fc85e41723e0e06", "value": " 3760/3760 [00:00<00:00, 669635.39 examples/s]" } }, "c4c50f17cd50456e8749134980953dcd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f18ce9c9c3614872b9c6f58b60320e17": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "67d1880b56f4410fa1107c05f30c403c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "8a0b7ed6756047aa99ff1a3742071450": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6fd164f995d943d1a50a3e41f134fdc2": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "4c8cdc6dce984019ac5a13e89f9c4f1b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "033bf4e55a0f46f59fc85e41723e0e06": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "cRPfm9MsgAs0", "outputId": "bee3c8d1-9367-466e-a95d-ab218706a5dd" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ single feat (8, 64) stages 1 params 1,248 dev0 +0.067 rank0 3.78\n", " ✓ sum feat (8, 64) stages 3 params 3,747 dev0 +0.067 rank0 3.78\n", " ✓ gate feat (8, 64) stages 3 params 3,905 dev0 +0.067 rank0 3.78\n", " ✓ res feat (8, 64) stages 3 params 5,280 dev0 +0.067 rank0 3.78\n", " ✓ prod feat (8, 64) stages 3 params 4,512 dev0 +0.067 rank0 3.78\n", " ✓ tree feat (8, 64) stages 2 params 2,976 dev0 +0.067 rank0 3.78\n", " ✓ tree(hard) feat (8, 64) stages 2 params 2,976 dev0 +0.067 rank0 3.78\n", " ✓ cross feat (8, 64) stages 3 params 6,096 dev0 +0.067 rank0 3.78\n", " ✓ anneal feat (8, 64) stages 3 params 3,424 dev0 +0.067 rank0 3.78\n", " ✓ spread construction: dev +0.078 rank 3.97 (polytope)\n", " ✓ budget matcher sane\n", "acd_structures smoke: ALL GREEN\n" ] } ], "source": [ "# ============================================================\n", "# acd_structures.py — exp_011 Additive-Conjunctive Differentiation\n", "# The seven composition operators over micro-alephs.\n", "#\n", "# Every operator composes m signed-projective addressers and exposes ONE\n", "# interface so the Forge can treat structures as interchangeable formulas:\n", "# forward(x) -> (features, addresses)\n", "# features : (B, F) — what the head reads (prediction closes\n", "# the loop THROUGH the structure)\n", "# addresses: (B, m, 2K_max) — per-stage p± (zero-padded to K_max),\n", "# for gauges only (no grad consumers)\n", "# stage_codebooks() -> list[Tensor (K_i, D_i)] — for statute gauges\n", "# param_count() -> int (codebook + structural params)\n", "#\n", "# Core math lifted VERBATIM from the trusted Jun-12 sources:\n", "# _address / _confidence <- aleph_routed_attention.py\n", "# _acd_max_spread_points <- aleph_lm.py\n", "# _acd_canon/_acd_mean_projective_angle/\n", "# acd_projective_deviation/statute <- aleph_trigram_lm.py\n", "#\n", "# Statutes: pure Adam upstream; clip max(loss,1.0); no GAP/BN/Dropout on\n", "# geometric paths; orthogonal init; forward paths return TENSOR TUPLES only\n", "# (never multi-key dicts — compile statute); gauges are separate methods.\n", "# ============================================================\n", "from __future__ import annotations\n", "import math\n", "from dataclasses import dataclass, field\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "OPS = (\"single\", \"sum\", \"gate\", \"res\", \"prod\", \"tree\", \"cross\", \"anneal\")\n", "\n", "@dataclass\n", "class ACDConfig:\n", " \"\"\"One composed structure. `single` (m=1) is the budget-matched baseline\n", " the Forge auto-inserts as every arm's twin.\"\"\"\n", " op: str = \"single\" # one of OPS\n", " d_in: int = 32 # input dimensionality\n", " m: int = 1 # number of stages / sub-alephs\n", " K: int = 16 # axes per stage (uniform v1)\n", " d_addr: int = 4 # address dim per stage (program home: 4)\n", " tau: float = 0.1 # address temperature (ANNEAL overrides per stage)\n", " freeze: str = \"free\" # 'free' | 'spread' | 'random' (exp_010 trio)\n", " feature_dim: int = 64 # head-facing feature width\n", " # op-specific knobs\n", " tree_hard: bool = False # TREE: top-1 + straight-through vs soft mixture\n", " tree_branch_k: int = 8 # TREE: K of the stage-0 (router) aleph\n", " cross_rank: int = 8 # CROSS: rank of factorized bilinear terms\n", " anneal_taus: Tuple[float, ...] = () # ANNEAL: explicit ladder; default geometric\n", " res_detach: bool = False # RES: detach residual between stages (RQ-style)\n", " seed: int = 1234\n", "\n", " def __post_init__(self):\n", " assert self.op in OPS, f\"unknown op {self.op}\"\n", " if self.op == \"single\":\n", " self.m = 1\n", " if self.op == \"anneal\" and not self.anneal_taus:\n", " # geometric ladder tau, tau/2, tau/4, ... (coarse -> fine)\n", " self.anneal_taus = tuple(self.tau / (2 ** t) for t in range(self.m))\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Statute gauges (verbatim program definitions)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _acd_canon(x: Tensor) -> Tensor:\n", " \"\"\"Sign-canonicalize onto RP^(D-1): flip so the first nonzero coord is\n", " positive (antipodes map to one representative).\"\"\"\n", " x = F.normalize(x, dim=-1)\n", " first_nz = x[torch.arange(len(x)), x.abs().argmax(dim=-1)]\n", " return x * torch.sign(first_nz).unsqueeze(-1)\n", "\n", "\n", "def _acd_mean_projective_angle(X: Tensor) -> float:\n", " \"\"\"Mean pairwise acos|cos| over distinct pairs (radians).\"\"\"\n", " c = (X @ X.t()).clamp(-1.0, 1.0).abs()\n", " iu = torch.triu_indices(len(X), len(X), offset=1)\n", " return torch.acos(c[iu[0], iu[1]]).mean().item()\n", "\n", "\n", "_ACD_UNIFORM_BASELINE: Dict[int, float] = {}\n", "\n", "def acd_projective_deviation(axes: Tensor, n_ref: int = 4096,\n", " seed: int = 0) -> float:\n", " \"\"\"Uniformity deviation per the program definition: mean pairwise\n", " projective angle of the axes MINUS the same statistic for n_ref uniform\n", " random projective points at the same D. Signed; sign matters.\"\"\"\n", " D = axes.shape[-1]\n", " if D not in _ACD_UNIFORM_BASELINE:\n", " g = torch.Generator().manual_seed(seed)\n", " ref = F.normalize(torch.randn(n_ref, D, generator=g), dim=-1)\n", " _ACD_UNIFORM_BASELINE[D] = _acd_mean_projective_angle(ref)\n", " return _acd_mean_projective_angle(F.normalize(axes.float(), dim=-1)) \\\n", " - _ACD_UNIFORM_BASELINE[D]\n", "\n", "\n", "def acd_statute(axes: Tensor) -> Dict[str, object]:\n", " \"\"\"Classify per the program taxonomy: dev > +0.05 polytope-class;\n", " |dev| < 0.05 uniform-class; dev < -0.05 degenerate (failure statute).\n", " NAMESPACE NOTE: prefixed acd_ — aleph_trigram_lm's `statute` must stay\n", " authoritative in the shared paste namespace (aleph_lm's logging uses it).\n", " Device-hardened: gauges always compute on a cpu copy (K x D is tiny).\"\"\"\n", " axes = axes.detach().float().cpu()\n", " dev = acd_projective_deviation(axes)\n", " A = F.normalize(axes.float(), dim=-1)\n", " sv = torch.linalg.svdvals(A)\n", " eff_rank = (sv.sum() ** 2 / (sv ** 2).sum()).item()\n", " c = (A @ A.t()).clamp(-1, 1).abs()\n", " c.fill_diagonal_(0.0)\n", " min_angle = math.degrees(math.acos(c.max().item()))\n", " cls = (\"polytope\" if dev > 0.05 else\n", " \"degenerate\" if dev < -0.05 else \"uniform\")\n", " return {\"deviation\": dev, \"eff_rank\": eff_rank,\n", " \"eff_rank_ceiling\": float(min(axes.shape)), # rank <= min(K, D)\n", " \"min_angle_deg\": min_angle, \"statute\": cls}\n", "\n", "\n", "def _acd_max_spread_points(K: int, D: int, seed: int = 0,\n", " iters: int = 2000) -> torch.Tensor:\n", " \"\"\"Deterministic maximal-spread K points on S^(D-1): seeded init +\n", " sign-symmetric repulsion (minimize sum exp of |cos|), Adam, renormalized.\n", " Statute-by-construction codebook for freeze='spread'.\"\"\"\n", " g = torch.Generator().manual_seed(seed)\n", " x = torch.nn.Parameter(F.normalize(torch.randn(K, D, generator=g), dim=-1))\n", " o = torch.optim.Adam([x], lr=0.05)\n", " eye = torch.eye(K, dtype=torch.bool)\n", " for beta in (8.0, 16.0, 32.0, 64.0):\n", " for _ in range(iters // 2):\n", " o.zero_grad()\n", " u = F.normalize(x, dim=-1)\n", " c = (u @ u.t()).masked_fill(eye, 0.0)\n", " loss = torch.logsumexp(beta * c.abs().flatten(), 0) / beta\n", " loss.backward()\n", " o.step()\n", " with torch.no_grad(): # saddle escape between rounds\n", " u = F.normalize(x, dim=-1)\n", " c = (u @ u.t()).masked_fill(eye, 0.0).abs()\n", " stuck = (c > 0.95).any(dim=1)\n", " if stuck.any():\n", " x[stuck] += 0.2 * torch.randn(int(stuck.sum()), D, generator=g)\n", " return F.normalize(x.detach(), dim=-1)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# MicroAleph — one stage: proj -> S^(D-1) -> signed-projective address\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class MicroAleph(nn.Module):\n", " \"\"\"One addresser. forward(x: (B, d_in)) -> (p_plus, p_minus): (B, K) each.\n", " Projection is orthogonal-init Linear -> sphere norm (the premise).\"\"\"\n", "\n", " def __init__(self, d_in: int, K: int, d_addr: int, tau: float,\n", " freeze: str, seed: int):\n", " super().__init__()\n", " self.K, self.Da, self.tau = K, d_addr, tau\n", " self.to_addr = nn.Linear(d_in, d_addr, bias=False)\n", " g = torch.Generator().manual_seed(seed)\n", " with torch.no_grad():\n", " w = torch.empty(d_addr, d_in)\n", " torch.nn.init.orthogonal_(w, gain=1.0)\n", " # seed-dependent orthogonal frame: permute rows deterministically\n", " w = w[torch.randperm(d_addr, generator=g)]\n", " self.to_addr.weight.copy_(w)\n", " if freeze == \"spread\":\n", " cb = _acd_max_spread_points(K, d_addr, seed=seed)\n", " elif freeze == \"random\":\n", " cb = F.normalize(torch.randn(K, d_addr, generator=g), dim=-1)\n", " else: # free\n", " cb = F.normalize(torch.randn(K, d_addr, generator=g), dim=-1)\n", " self.codebook = nn.Parameter(cb, requires_grad=(freeze == \"free\"))\n", " self.freeze = freeze\n", "\n", " def _address(self, x_hat: Tensor) -> Tuple[Tensor, Tensor]:\n", " \"\"\"Aleph address of unit rows x_hat (..., Da) against the codebook.\n", " Exact softmax over the 2K oriented axes, antipodally factored; the 2K\n", " tensor is never materialized. Stable via max|u| subtraction.\"\"\"\n", " A = F.normalize(self.codebook, dim=-1) # (K, Da)\n", " u = (x_hat @ A.t()) * (1.0 / self.tau) # (..., K) signed\n", " m = u.abs().amax(dim=-1, keepdim=True)\n", " ep = torch.exp(u - m) # ∝ e^{+u}\n", " en = torch.exp(-u - m) # ∝ e^{-u}\n", " Z = (ep + en).sum(dim=-1, keepdim=True) # >= 1 by construction\n", " return ep / Z, en / Z\n", "\n", " def forward(self, x: Tensor, tau_override: Optional[float] = None\n", " ) -> Tuple[Tensor, Tensor]:\n", " x_hat = F.normalize(self.to_addr(x), dim=-1)\n", " if tau_override is not None:\n", " saved, self.tau = self.tau, tau_override\n", " out = self._address(x_hat)\n", " self.tau = saved\n", " return out\n", " return self._address(x_hat)\n", "\n", " def confidence(self, p: Tensor, n: Tensor) -> Tensor:\n", " \"\"\"||(p+ - p-) @ A|| in (0, 1] — norm of the soft codebook\n", " reconstruction (verbatim program definition).\"\"\"\n", " A = F.normalize(self.codebook, dim=-1)\n", " return ((p - n) @ A).norm(dim=-1)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# ACDStructure — the seven operators behind one interface\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class ACDStructure(nn.Module):\n", " \"\"\"Composed addresser. features = W_f · concat(stage vectors); each stage\n", " vector = [p+ | p- | confidence] (2K+1). The head reads ONLY features, so\n", " prediction pressure flows through every stage that contributes —\n", " the employment law, enforced architecturally.\"\"\"\n", "\n", " def __init__(self, cfg: ACDConfig):\n", " super().__init__()\n", " self.cfg = cfg\n", " c = cfg\n", " torch.manual_seed(c.seed)\n", "\n", " if c.op == \"tree\":\n", " # stage 0 = router aleph (K = tree_branch_k); branches B = 2*K0\n", " # (oriented axes ARE the branches — signed structure used as the\n", " # decision variable). Stage 1 = per-branch codebooks (B, K, Da).\n", " self.router = MicroAleph(c.d_in, c.tree_branch_k, c.d_addr,\n", " c.tau, c.freeze, c.seed)\n", " B = 2 * c.tree_branch_k\n", " if c.freeze == \"spread\":\n", " cb = torch.stack([_acd_max_spread_points(c.K, c.d_addr, seed=c.seed + 1 + b)\n", " for b in range(B)])\n", " else:\n", " g = torch.Generator().manual_seed(c.seed + 1)\n", " cb = F.normalize(torch.randn(B, c.K, c.d_addr, generator=g), dim=-1)\n", " self.branch_books = nn.Parameter(cb, requires_grad=(c.freeze == \"free\"))\n", " self.branch_proj = nn.Linear(c.d_in, c.d_addr, bias=False)\n", " torch.nn.init.orthogonal_(self.branch_proj.weight)\n", " self.n_stage_vec = 2 # router vec + branch vec\n", " stage_widths = [2 * c.tree_branch_k + 1, 2 * c.K + 1]\n", " else:\n", " n = c.m if c.op != \"single\" else 1\n", " self.stages = nn.ModuleList(\n", " MicroAleph(c.d_in, c.K, c.d_addr, c.tau, c.freeze, c.seed + i)\n", " for i in range(n))\n", " self.n_stage_vec = n\n", " stage_widths = [2 * c.K + 1] * n\n", " if c.op == \"sum\":\n", " self.mix_logits = nn.Parameter(torch.zeros(n))\n", " if c.op == \"gate\":\n", " self.gater = MicroAleph(c.d_in, max(n, 2), c.d_addr,\n", " c.tau, c.freeze, c.seed + 977)\n", " self.gate_read = nn.Linear(2 * max(n, 2) + 1, n, bias=False)\n", " torch.nn.init.orthogonal_(self.gate_read.weight)\n", " if c.op == \"res\":\n", " # decode each stage's address back to input space to subtract\n", " self.decoders = nn.ModuleList(\n", " nn.Linear(2 * c.K, c.d_in, bias=False) for _ in range(n))\n", " for d in self.decoders:\n", " torch.nn.init.orthogonal_(d.weight)\n", " if c.op == \"prod\":\n", " self.rot = nn.Linear(c.d_in, c.d_in, bias=False)\n", " torch.nn.init.orthogonal_(self.rot.weight)\n", " # each stage sees one contiguous subspace slice\n", " self.slices = [(i * c.d_in // n, (i + 1) * c.d_in // n)\n", " for i in range(n)]\n", " # rebuild stages with per-slice input widths\n", " self.stages = nn.ModuleList(\n", " MicroAleph(hi - lo, c.K, c.d_addr, c.tau, c.freeze,\n", " c.seed + i)\n", " for i, (lo, hi) in enumerate(self.slices))\n", " if c.op == \"cross\":\n", " r = c.cross_rank\n", " self.cross_l = nn.ModuleList(\n", " nn.Linear(2 * c.K + 1, r, bias=False) for _ in range(n))\n", " self.cross_r = nn.ModuleList(\n", " nn.Linear(2 * c.K + 1, r, bias=False) for _ in range(n))\n", " for lin in (*self.cross_l, *self.cross_r):\n", " torch.nn.init.orthogonal_(lin.weight)\n", " n_pairs = n * (n - 1) // 2\n", " stage_widths = stage_widths + [c.cross_rank] * n_pairs\n", " if c.op == \"anneal\":\n", " # ONE codebook, m temperature rungs\n", " self.stages = nn.ModuleList([\n", " MicroAleph(c.d_in, c.K, c.d_addr, c.tau, c.freeze, c.seed)])\n", "\n", " self.K_max = max(c.K, getattr(c, \"tree_branch_k\", c.K))\n", " self.head_in = sum(stage_widths)\n", " self.to_features = nn.Linear(self.head_in, c.feature_dim, bias=False)\n", " torch.nn.init.orthogonal_(self.to_features.weight)\n", "\n", " # ---- helpers -----------------------------------------------------\n", " @staticmethod\n", " def _vec(p: Tensor, n: Tensor, conf: Tensor) -> Tensor:\n", " return torch.cat([p, n, conf.unsqueeze(-1)], dim=-1)\n", "\n", " def _pad_addr(self, p: Tensor, n: Tensor) -> Tensor:\n", " \"\"\"(B,K)+(B,K) -> (B, 2*K_max) zero-padded, for the gauge stack.\"\"\"\n", " a = torch.cat([p, n], dim=-1)\n", " pad = 2 * self.K_max - a.shape[-1]\n", " return F.pad(a, (0, pad)) if pad > 0 else a\n", "\n", " # ---- the operators ----------------------------------------------\n", " def forward(self, x: Tensor) -> Tuple[Tensor, Tensor]:\n", " c = self.cfg\n", " vecs: List[Tensor] = []\n", " addrs: List[Tensor] = []\n", "\n", " if c.op in (\"single\", \"sum\", \"gate\"):\n", " ps, ns, cs = [], [], []\n", " for st in self.stages:\n", " p, n = st(x)\n", " ps.append(p); ns.append(n); cs.append(st.confidence(p, n))\n", " addrs.append(self._pad_addr(p, n))\n", " if c.op == \"single\":\n", " vecs.append(self._vec(ps[0], ns[0], cs[0]))\n", " elif c.op == \"sum\":\n", " w = torch.softmax(self.mix_logits, dim=0) # UNCONDITIONED\n", " p = sum(w[i] * ps[i] for i in range(len(ps)))\n", " n = sum(w[i] * ns[i] for i in range(len(ns)))\n", " conf = sum(w[i] * cs[i] for i in range(len(cs)))\n", " # head reads only the SUM — stages have no private channel:\n", " # this is the control's defining austerity.\n", " vecs.append(F.pad(self._vec(p, n, conf),\n", " (0, self.head_in - (2 * c.K + 1))))\n", " else: # gate — input-dependent adjudication\n", " gp, gn = self.gater(x)\n", " g = torch.softmax(\n", " self.gate_read(self._vec(gp, gn,\n", " self.gater.confidence(gp, gn))),\n", " dim=-1) # (B, m)\n", " p = sum(g[:, i:i+1] * ps[i] for i in range(len(ps)))\n", " n = sum(g[:, i:i+1] * ns[i] for i in range(len(ns)))\n", " conf = sum(g[:, i] * cs[i] for i in range(len(cs)))\n", " vecs.append(F.pad(self._vec(p, n, conf),\n", " (0, self.head_in - (2 * c.K + 1))))\n", "\n", " elif c.op == \"res\":\n", " r = x\n", " for t, st in enumerate(self.stages):\n", " p, n = st(r)\n", " conf = st.confidence(p, n)\n", " vecs.append(self._vec(p, n, conf))\n", " addrs.append(self._pad_addr(p, n))\n", " recon = self.decoders[t](torch.cat([p, n], dim=-1))\n", " r = (r - recon).detach() if c.res_detach else r - recon\n", "\n", " elif c.op == \"prod\":\n", " xr = self.rot(x)\n", " for st, (lo, hi) in zip(self.stages, self.slices):\n", " p, n = st(xr[:, lo:hi])\n", " vecs.append(self._vec(p, n, st.confidence(p, n)))\n", " addrs.append(self._pad_addr(p, n))\n", "\n", " elif c.op == \"tree\":\n", " rp, rn = self.router(x) # (B, K0)\n", " rconf = self.router.confidence(rp, rn)\n", " vecs.append(self._vec(rp, rn, rconf))\n", " addrs.append(self._pad_addr(rp, rn))\n", " w = torch.cat([rp, rn], dim=-1) # (B, B_br)\n", " if c.tree_hard:\n", " idx = w.argmax(dim=-1)\n", " hard = F.one_hot(idx, w.shape[-1]).to(w.dtype)\n", " w = hard + (w - w.detach()) # straight-through\n", " x_hat = F.normalize(self.branch_proj(x), dim=-1) # (B, Da)\n", " A = F.normalize(self.branch_books, dim=-1) # (Bb, K, Da)\n", " u = torch.einsum(\"bd,ckd->bck\", x_hat, A) / self.cfg.tau\n", " mm = u.abs().amax(dim=(-1, -2), keepdim=True)\n", " ep, en = torch.exp(u - mm), torch.exp(-u - mm)\n", " Z = (ep + en).sum(dim=-1, keepdim=True)\n", " pb, nb = ep / Z, en / Z # (B, Bb, K)\n", " p = torch.einsum(\"bc,bck->bk\", w, pb) # branch-mixed\n", " n = torch.einsum(\"bc,bck->bk\", w, nb)\n", " conf = (p - n).norm(dim=-1) # soft-recon proxy\n", " vecs.append(self._vec(p, n, conf))\n", " addrs.append(self._pad_addr(p, n))\n", "\n", " elif c.op == \"cross\":\n", " base = []\n", " for st in self.stages:\n", " p, n = st(x)\n", " v = self._vec(p, n, st.confidence(p, n))\n", " base.append(v); vecs.append(v)\n", " addrs.append(self._pad_addr(p, n))\n", " for i in range(len(base)):\n", " for j in range(i + 1, len(base)):\n", " vecs.append(self.cross_l[i](base[i]) *\n", " self.cross_r[j](base[j])) # factorized ⊗\n", "\n", " elif c.op == \"anneal\":\n", " st = self.stages[0]\n", " for t in range(c.m):\n", " p, n = st(x, tau_override=c.anneal_taus[t])\n", " vecs.append(self._vec(p, n, st.confidence(p, n)))\n", " addrs.append(self._pad_addr(p, n))\n", "\n", " features = self.to_features(torch.cat(vecs, dim=-1))\n", " addresses = torch.stack(addrs, dim=1) # (B, m', 2K_max)\n", " return features, addresses\n", "\n", " # ---- gauges (non-compiled surface; dicts allowed here) -----------\n", " def stage_codebooks(self) -> List[Tensor]:\n", " c = self.cfg\n", " if c.op == \"tree\":\n", " books = [self.router.codebook.detach()]\n", " books += [self.branch_books[b].detach()\n", " for b in range(self.branch_books.shape[0])]\n", " return books\n", " return [st.codebook.detach() for st in self.stages]\n", "\n", " def codebook_stats(self) -> List[Dict[str, object]]:\n", " return [acd_statute(cb) for cb in self.stage_codebooks()]\n", "\n", " def param_count(self) -> int:\n", " return sum(p.numel() for p in self.parameters())\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Budget matching — solve per-stage K so total codebook params match target\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def match_budget(op: str, m: int, d_addr: int, budget_kd: int,\n", " tree_branch_k: int = 8) -> int:\n", " \"\"\"Return per-stage K such that total codebook floats ≈ budget_kd\n", " (= K_ref * D_ref of the single-aleph reference). TREE counts router +\n", " 2*K0 branch books.\"\"\"\n", " if op in (\"single\",):\n", " return max(2, budget_kd // d_addr)\n", " if op == \"anneal\":\n", " return max(2, budget_kd // d_addr) # one shared book\n", " if op == \"tree\":\n", " remaining = budget_kd - tree_branch_k * d_addr\n", " return max(2, remaining // (2 * tree_branch_k * d_addr))\n", " return max(2, budget_kd // (m * d_addr)) # sum/gate/res/prod/cross\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke — runs on import guard; every operator fwd/bwd + gauge sanity\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " torch.manual_seed(0)\n", " B, d_in = 8, 32\n", " x = torch.randn(B, d_in)\n", " for op in OPS:\n", " for hard in ([False, True] if op == \"tree\" else [False]):\n", " cfg = ACDConfig(op=op, d_in=d_in, m=3, K=8, d_addr=4,\n", " tree_hard=hard, freeze=\"free\")\n", " net = ACDStructure(cfg)\n", " f, a = net(x)\n", " assert f.shape == (B, cfg.feature_dim), (op, f.shape)\n", " assert torch.isfinite(f).all() and torch.isfinite(a).all(), op\n", " # address rows on padded 2K_max simplex: mass ≤ 1, ≥ 0\n", " assert (a >= -1e-6).all() and a.sum(-1).max() <= 1.0 + 1e-4, op\n", " loss = f.square().mean()\n", " loss.backward()\n", " grads = [p.grad for p in net.parameters()\n", " if p.requires_grad and p.grad is not None]\n", " assert grads and all(torch.isfinite(g).all() for g in grads), op\n", " st = net.codebook_stats()\n", " assert all(\"deviation\" in s for s in st)\n", " tag = f\"{op}{'(hard)' if hard else ''}\"\n", " print(f\" ✓ {tag:12s} feat {tuple(f.shape)} stages {a.shape[1]}\"\n", " f\" params {net.param_count():,}\"\n", " f\" dev0 {st[0]['deviation']:+.3f} rank0 {st[0]['eff_rank']:.2f}\")\n", " # spread construction pins rank/statute\n", " cfg = ACDConfig(op=\"single\", d_in=d_in, K=8, d_addr=4, freeze=\"spread\")\n", " st = ACDStructure(cfg).codebook_stats()[0]\n", " assert st[\"statute\"] == \"polytope\" and st[\"eff_rank\"] > 3.9, st\n", " print(f\" ✓ spread construction: dev {st['deviation']:+.3f} \"\n", " f\"rank {st['eff_rank']:.2f} ({st['statute']})\")\n", " # budget matcher: composed ops land within one stage-row of target\n", " for op in (\"sum\", \"res\", \"prod\", \"tree\"):\n", " K = match_budget(op, 4, 4, budget_kd=64 * 4)\n", " assert K >= 2, (op, K)\n", " print(\" ✓ budget matcher sane\")\n", " print(\"acd_structures smoke: ALL GREEN\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " # Notebook cells execute as __main__, so the smoke fires on paste too —\n", " # deliberate: pasting a cell IS the verification step in the Colab flow\n", " # (shared namespace, paste order structures -> probe -> forge).\n", " # Heavy entry points (phase2_screen) are never wired here; call them.\n", " _smoke()" ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_probe.py — exp_011 Tier-P instrument\n", "# Nested globular clusters (the VAE-bubble picture, made exact) +\n", "# the marginal-bits methodology + composition gauges.\n", "#\n", "# Ground truth is generative, so per-level information is EXACT:\n", "# available bits at level l = log2(branching_l)\n", "# total = log2(#leaves)\n", "# The staged-probe estimator Î(Y; A_t | A_ acd_probe.py -> acd_forge.py\n", "# ============================================================\n", "from __future__ import annotations\n", "import math\n", "from dataclasses import dataclass\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure # module mode\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class BubbleConfig:\n", " \"\"\"Nested Gaussian bubbles. Level-l centers sit on a sphere of radius\n", " sep_l around their parent; sep decays geometrically so structure is\n", " coarse-to-fine, matching the differentiation story.\"\"\"\n", " d_data: int = 32\n", " branching: Tuple[int, ...] = (4, 4, 4) # per-level fanout; leaves = prod\n", " sep0: float = 6.0 # level-1 shell radius\n", " sep_decay: float = 0.45 # sep_{l+1} = sep_l * decay\n", " noise: float = 0.35 # leaf-local sigma\n", " seed: int = 1234\n", "\n", " @property\n", " def n_leaves(self) -> int:\n", " n = 1\n", " for b in self.branching:\n", " n *= b\n", " return n\n", "\n", " @property\n", " def bits_per_level(self) -> List[float]:\n", " return [math.log2(b) for b in self.branching]\n", "\n", "\n", "class NestedBubbles:\n", " \"\"\"Sampler + exact labels. sample(n) -> (x, leaf, levels) where\n", " levels[:, l] is the level-l ancestor index (the 'subsequent\n", " differentiation' ladder).\"\"\"\n", "\n", " def __init__(self, cfg: BubbleConfig):\n", " self.cfg = cfg\n", " g = torch.Generator().manual_seed(cfg.seed)\n", " centers = [torch.zeros(1, cfg.d_data)]\n", " sep = cfg.sep0\n", " for b in cfg.branching:\n", " parent = centers[-1] # (P, d)\n", " offs = F.normalize(torch.randn(parent.shape[0], b, cfg.d_data,\n", " generator=g), dim=-1) * sep\n", " centers.append((parent.unsqueeze(1) + offs).reshape(-1, cfg.d_data))\n", " sep *= cfg.sep_decay\n", " self.level_centers = centers[1:] # per level\n", " self.leaf_centers = centers[-1] # (n_leaves, d)\n", " self._g = g\n", "\n", " def sample(self, n: int, device=\"cpu\") -> Tuple[Tensor, Tensor, Tensor]:\n", " c = self.cfg\n", " leaf = torch.randint(0, c.n_leaves, (n,), generator=self._g)\n", " x = self.leaf_centers[leaf] + c.noise * torch.randn(\n", " n, c.d_data, generator=self._g)\n", " # ancestor index at level l = leaf // prod(branching[l+1:])\n", " levels = []\n", " div = 1\n", " for b in reversed(c.branching):\n", " levels.append(leaf // div)\n", " div *= b\n", " levels = torch.stack(list(reversed(levels)), dim=1) # (n, L) coarse->fine\n", " return x.to(device), leaf.to(device), levels.to(device)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Marginal-bits estimator — staged probes\n", "# Î(Y; A_t | A_ y on a held-out half.\n", " Pure Adam, standing clip rule.\"\"\"\n", " dev = feats.device\n", " g = torch.Generator(device=\"cpu\").manual_seed(seed)\n", " n = feats.shape[0]\n", " perm = torch.randperm(n, generator=g).to(dev)\n", " tr, te = perm[: n // 2], perm[n // 2:]\n", " probe = nn.Linear(feats.shape[1], n_classes).to(dev)\n", " opt = torch.optim.Adam(probe.parameters(), lr=lr,\n", " weight_decay=weight_decay)\n", " ftr, ytr = feats[tr].detach(), y[tr]\n", " for _ in range(steps):\n", " opt.zero_grad()\n", " loss = F.cross_entropy(probe(ftr), ytr)\n", " loss.backward()\n", " torch.nn.utils.clip_grad_norm_(probe.parameters(),\n", " max(loss.item(), 1.0))\n", " opt.step()\n", " with torch.no_grad():\n", " logits = probe(feats[te].detach())\n", " ce = F.cross_entropy(logits, y[te]).item()\n", " if return_acc:\n", " acc = (logits.argmax(-1) == y[te]).float().mean().item()\n", " return ce / math.log(2.0), acc\n", " return ce / math.log(2.0)\n", "\n", "\n", "@torch.no_grad()\n", "def _collect_addresses(structure, x: Tensor) -> Tensor:\n", " structure.eval()\n", " _, addrs = structure(x) # (B, m, 2K_max)\n", " return addrs\n", "\n", "\n", "def marginal_bits(structure, x: Tensor, y: Tensor, n_classes: int,\n", " probe_steps: int = 300, seed: int = 0) -> Dict[str, object]:\n", " \"\"\"The headline gauge. Returns per-stage marginal bits, cumulative\n", " curve, and the entropy floor reached. H(Y|empty) = log2(n_classes)\n", " for the uniform sampler (exact, by construction).\"\"\"\n", " addrs = _collect_addresses(structure, x) # (B, m, W)\n", " B, m, W = addrs.shape\n", " H_prev = math.log2(n_classes) # exact prior\n", " curve, marginals = [], []\n", " for t in range(m):\n", " prefix = addrs[:, : t + 1].reshape(B, -1)\n", " H_t = _probe_ce_bits(prefix, y, n_classes,\n", " steps=probe_steps, seed=seed + t)\n", " H_t = min(H_t, H_prev) # information can't be destroyed\n", " marginals.append(H_prev - H_t)\n", " curve.append(math.log2(n_classes) - H_t)\n", " H_prev = H_t\n", " return {\"marginal_bits\": marginals, \"cumulative_bits\": curve,\n", " \"H_final\": H_prev, \"H_prior\": math.log2(n_classes)}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Composition gauges — redundancy, cancellation, stage SNR\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@torch.no_grad()\n", "def composition_gauges(structure, x: Tensor,\n", " cluster: Optional[Tensor] = None) -> Dict[str, float]:\n", " \"\"\"redundancy: mean pairwise cosine between stage address distributions\n", " (0 = independent stages; 1 = clones — the SUM failure signature).\n", " cancellation: mass-weighted sign-conflict rate on shared axes\n", " (meaningful for SUM/GATE/ANNEAL which share axis space; reported for\n", " all, interpret per-op).\n", " stage_snr: between-cluster / within-cluster variance of the ACCUMULATED\n", " address (needs cluster labels; the divergence detector).\"\"\"\n", " addrs = _collect_addresses(structure, x) # (B, m, W)\n", " B, m, W = addrs.shape\n", " K = W // 2\n", " out: Dict[str, float] = {}\n", " # redundancy\n", " if m > 1:\n", " flat = F.normalize(addrs, dim=-1) # (B, m, W)\n", " cos = torch.einsum(\"bmw,bnw->bmn\", flat, flat)\n", " iu = torch.triu_indices(m, m, offset=1)\n", " out[\"redundancy\"] = cos[:, iu[0], iu[1]].mean().item()\n", " else:\n", " out[\"redundancy\"] = 0.0\n", " # hemisphere cancellation: stage-wise net vote per axis = p+ - p-;\n", " # conflict where signs disagree across stages, weighted by joint mass\n", " if m > 1:\n", " net = addrs[..., :K] - addrs[..., K:] # (B, m, K)\n", " sign = net.sign()\n", " agree = (sign.sum(dim=1).abs() == m).float() # all stages agree\n", " mass = addrs.sum(dim=(1, 2)).clamp_min(1e-9)\n", " conflict_mass = ((1 - agree) * net.abs().sum(dim=1)).sum(-1)\n", " out[\"cancellation\"] = (conflict_mass /\n", " net.abs().sum(dim=(1, 2)).clamp_min(1e-9)\n", " ).mean().item()\n", " else:\n", " out[\"cancellation\"] = 0.0\n", " # stage SNR on accumulated address\n", " if cluster is not None:\n", " acc = addrs.reshape(B, -1)\n", " mu = acc.mean(0)\n", " between, within, nc = 0.0, 0.0, 0\n", " for c in cluster.unique():\n", " sel = acc[cluster == c]\n", " if len(sel) < 2:\n", " continue\n", " between += (sel.mean(0) - mu).square().sum().item() * len(sel)\n", " within += (sel - sel.mean(0)).square().sum().item()\n", " nc += len(sel)\n", " out[\"stage_snr\"] = between / max(within, 1e-9)\n", " return out\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Oracle / noise addressers — estimator calibration (Phase-1 gate)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class _OracleAddresser(nn.Module):\n", " \"\"\"Emits the TRUE level-t label one-hot as stage-t address. The\n", " marginal-bits estimator must recover ~log2(branching_t) per stage.\"\"\"\n", "\n", " def __init__(self, bubbles: NestedBubbles):\n", " super().__init__()\n", " self.bub = bubbles\n", " self.K_max = max(bubbles.cfg.branching)\n", " self._levels: Optional[Tensor] = None\n", "\n", " def bind(self, levels: Tensor):\n", " self._levels = levels\n", "\n", " def forward(self, x: Tensor) -> Tuple[Tensor, Tensor]:\n", " L = len(self.bub.cfg.branching)\n", " outs = []\n", " for t in range(L):\n", " oh = F.one_hot(self._levels[:, t] % self.bub.cfg.branching[t],\n", " self.K_max).float()\n", " outs.append(F.pad(oh, (0, self.K_max))) # p- pad\n", " a = torch.stack(outs, dim=1) # (B, L, 2K_max)\n", " return a.sum(1), a # features unused\n", "\n", " def eval(self):\n", " return self\n", "\n", "\n", "class _NoiseAddresser(_OracleAddresser):\n", " \"\"\"Random simplex mass — estimator must recover ~0 bits/stage.\"\"\"\n", "\n", " def forward(self, x: Tensor) -> Tuple[Tensor, Tensor]:\n", " L = len(self.bub.cfg.branching)\n", " a = torch.rand(x.shape[0], L, 2 * self.K_max, device=x.device)\n", " a = a / a.sum(-1, keepdim=True)\n", " return a.sum(1), a\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke — bubble geometry, estimator calibration, gauge sanity\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " torch.manual_seed(0)\n", " cfg = BubbleConfig(d_data=32, branching=(4, 4), sep0=6.0,\n", " noise=0.3, seed=7)\n", " bub = NestedBubbles(cfg)\n", " x, leaf, levels = bub.sample(3000)\n", " assert x.shape == (3000, 32) and leaf.max() < cfg.n_leaves\n", " # coarse structure dominates: level-1 centroid spread > level-2\n", " d1 = bub.level_centers[0].norm(dim=-1).mean().item()\n", " d2 = (bub.leaf_centers - bub.level_centers[0].repeat_interleave(\n", " cfg.branching[1], 0)).norm(dim=-1).mean().item()\n", " assert d1 > d2, (d1, d2)\n", " print(f\" ✓ bubbles: {cfg.n_leaves} leaves, shell radii {d1:.2f} > {d2:.2f}\")\n", "\n", " # ORACLE calibration: must recover ~log2(4)=2.0 bits per stage\n", " oracle = _OracleAddresser(bub); oracle.bind(levels)\n", " mb = marginal_bits(oracle, x, leaf, cfg.n_leaves, probe_steps=400)\n", " tgt = cfg.bits_per_level\n", " ok = all(abs(m - t) < 0.25 for m, t in zip(mb[\"marginal_bits\"], tgt))\n", " print(f\" ✓ oracle marginal bits {['%.2f' % m for m in mb['marginal_bits']]}\"\n", " f\" vs exact {tgt} -> {'PASS' if ok else 'FAIL'}\")\n", " assert ok, mb\n", "\n", " # NOISE calibration: ~0 bits per stage\n", " noise = _NoiseAddresser(bub)\n", " mbn = marginal_bits(noise, x, leaf, cfg.n_leaves, probe_steps=400)\n", " assert all(m < 0.15 for m in mbn[\"marginal_bits\"]), mbn\n", " print(f\" ✓ noise marginal bits {['%.2f' % m for m in mbn['marginal_bits']]}\"\n", " f\" (all ~0)\")\n", "\n", " # gauges run on a real structure\n", " s = ACDStructure(ACDConfig(op=\"sum\", d_in=32, m=3, K=8, d_addr=4))\n", " g = composition_gauges(s, x[:512], cluster=levels[:512, 0])\n", " assert set(g) >= {\"redundancy\", \"cancellation\", \"stage_snr\"}\n", " print(f\" ✓ gauges: red {g['redundancy']:.3f} canc {g['cancellation']:.3f}\"\n", " f\" snr {g['stage_snr']:.3f}\")\n", " print(\"acd_probe smoke: ALL GREEN\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " # Notebook cells execute as __main__, so the smoke fires on paste too —\n", " # deliberate: pasting a cell IS the verification step in the Colab flow\n", " # (shared namespace, paste order structures -> probe -> forge).\n", " # Heavy entry points (phase2_screen) are never wired here; call them.\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "IRYC0aSWgJrE", "outputId": "ea0a7033-1b54-45ae-c274-b348629f67d8" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ bubbles: 16 leaves, shell radii 6.00 > 2.70\n", " ✓ oracle marginal bits ['1.98', '1.99'] vs exact [2.0, 2.0] -> PASS\n", " ✓ noise marginal bits ['0.00', '0.00'] (all ~0)\n", " ✓ gauges: red 0.209 canc 0.552 snr 1.866\n", "acd_probe smoke: ALL GREEN\n" ] } ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_forge.py — exp_011 automation\n", "# Grammar -> generator (auto budget-twins + SUM controls) -> rung\n", "# scheduler (successive halving) -> kill rules -> ledger -> HF push.\n", "#\n", "# The Captain reviews VERDICTS, not arms: every promote/park/kill is\n", "# logged with the gauge values that caused it.\n", "#\n", "# Rungs v1: P-200 -> P-1000 (Tier-P implemented). Tier-L rungs raise\n", "# NotImplementedError at a clean seam until acd_lm_adapter.py lands.\n", "# Lane parallelism (vmap seed-groups): DEFERRED v1.1 — sequential is\n", "# correct and rung0 runs single-seed; the bit-equivalence gate applies\n", "# when lanes ship, not before.\n", "#\n", "# Repo: AbstractPhil/geolip-aleph-differentiation (exp011/ prefix)\n", "# Paste order: acd_structures.py -> acd_probe.py -> acd_forge.py\n", "# ============================================================\n", "from __future__ import annotations\n", "import csv, hashlib, json, math, os, time\n", "from dataclasses import asdict, dataclass, field\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure, match_budget, OPS\n", " from acd_probe import (BubbleConfig, NestedBubbles, marginal_bits,\n", " composition_gauges)\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver. Pasted Colab cells share ONE namespace and are\n", " not importable modules — so resolve names from globals() first (paste\n", " mode), then fall back to a real import (script/module mode).\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "REPO_ID = \"AbstractPhil/geolip-aleph-differentiation\"\n", "EXP_PREFIX = \"exp011\"\n", "BUDGET_KD = 64 * 4 # single-aleph reference: K=64, D=4 codebook floats\n", "\n", "@dataclass\n", "class ForgeConfig:\n", " out_dir: str = \"exp011\"\n", " rungs: Tuple[Tuple[str, int, float], ...] = (\n", " (\"P\", 200, 1 / 3), (\"P\", 1000, 1 / 3),\n", " (\"L\", 1000, 1 / 3), (\"L\", 5000, 1.0))\n", " lr: float = 3e-3\n", " batch: int = 512\n", " eval_every: int = 50\n", " probe_steps: int = 250 # marginal-bits probe budget per prefix\n", " n_train: int = 8192 # fixed shared dataset per screen\n", " n_eval: int = 4096\n", " bubble: \"BubbleConfig\" = None # set in __post_init__ (shared task!)\n", " push: bool = True # False = dry run (no network)\n", " lm: Optional[Dict] = None # Tier-L AlephLM overrides (None -> tier_l recipe)\n", " lm_corpus: str = \"wikitext-103-raw-v1\" # 'synthetic' -> Markov smoke stream\n", " device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " seed: int = 1234\n", "\n", " exp_prefix: str = \"\" # HF push prefix; defaults to out_dir\n", "\n", " def __post_init__(self):\n", " if not self.exp_prefix:\n", " self.exp_prefix = self.out_dir\n", " if self.bubble is None:\n", " self.bubble = BubbleConfig(d_data=32, branching=(4, 4, 4),\n", " sep0=6.0, sep_decay=0.45,\n", " noise=0.35, seed=97)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Grammar — an arm is a JSON spec; its hash is its identity\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class ArmSpec:\n", " op: str\n", " m: int\n", " d_addr: int = 4\n", " freeze: str = \"free\"\n", " seed: int = 1234\n", " tree_hard: bool = False\n", " budget_kd: int = BUDGET_KD\n", " K: int = 0 # 0 -> solved by match_budget\n", "\n", " def resolve(self) -> \"ArmSpec\":\n", " if self.K == 0:\n", " self.K = match_budget(self.op, self.m, self.d_addr,\n", " self.budget_kd)\n", " if self.op == \"single\":\n", " self.m = 1\n", " return self\n", "\n", " def arm_id(self) -> str:\n", " s = json.dumps(asdict(self), sort_keys=True)\n", " return f\"{self.op}_m{self.m}_K{self.K}\" \\\n", " f\"{'_hard' if self.tree_hard else ''}\" \\\n", " f\"_{self.freeze}_s{self.seed}_{hashlib.sha1(s.encode()).hexdigest()[:6]}\"\n", "\n", " def to_acd(self, d_in: int, feature_dim: int = 64) -> \"ACDConfig\":\n", " return ACDConfig(op=self.op, d_in=d_in, m=self.m, K=self.K,\n", " d_addr=self.d_addr, freeze=self.freeze,\n", " tree_hard=self.tree_hard, seed=self.seed,\n", " feature_dim=feature_dim)\n", "\n", "\n", "def generate_arms(ops: List[str], ms: List[int], freezes: List[str],\n", " seeds: List[int], existing_ids: Optional[set] = None,\n", " tree_both_modes: bool = True) -> List[ArmSpec]:\n", " \"\"\"Grid expansion + the two mandatory scientific controls:\n", " (1) budget-matched SINGLE twin per (freeze, seed),\n", " (2) SUM control at every m present (the divergence reference).\n", " Dedup against existing ledger ids.\"\"\"\n", " arms: List[ArmSpec] = []\n", " for op in ops:\n", " for m in ms:\n", " if op == \"single\":\n", " continue\n", " for fz in freezes:\n", " for sd in seeds:\n", " if op == \"tree\" and tree_both_modes:\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd,\n", " tree_hard=False))\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd,\n", " tree_hard=True))\n", " else:\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd))\n", " for fz in freezes: # control (1)\n", " for sd in seeds:\n", " arms.append(ArmSpec(\"single\", 1, freeze=fz, seed=sd))\n", " if \"sum\" not in ops: # control (2)\n", " for m in ms:\n", " for fz in freezes:\n", " for sd in seeds:\n", " arms.append(ArmSpec(\"sum\", m, freeze=fz, seed=sd))\n", " out, seen = [], set(existing_ids or ())\n", " for a in arms:\n", " a.resolve()\n", " if a.arm_id() not in seen:\n", " seen.add(a.arm_id())\n", " out.append(a)\n", " return out\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Kill rules — fire inside a rung; every kill carries its reason\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def check_kill(loss_val: float, grad_norm: float,\n", " stats: List[Dict]) -> Optional[str]:\n", " if not math.isfinite(loss_val):\n", " return \"NaN/inf loss\"\n", " if grad_norm > 1e3:\n", " return f\"grad blowup |g|={grad_norm:.1f}\"\n", " for i, s in enumerate(stats):\n", " ceiling = min(4.0, s.get(\"eff_rank_ceiling\", 4.0))\n", " if s[\"eff_rank\"] < 0.5 * ceiling:\n", " return f\"rank collapse stage{i} rank={s['eff_rank']:.2f}\"\n", " return None\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier-P trainer — one arm, one rung\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_arm_P(spec: ArmSpec, steps: int, fc: ForgeConfig,\n", " data: Tuple) -> Dict[str, object]:\n", " xtr, ytr, ltr, xev, yev, lev = data\n", " dev = fc.device\n", " torch.manual_seed(spec.seed)\n", " net = ACDStructure(spec.to_acd(d_in=fc.bubble.d_data)).to(dev)\n", " head = nn.Linear(net.cfg.feature_dim, fc.bubble.n_leaves).to(dev)\n", " opt = torch.optim.Adam(list(net.parameters()) + list(head.parameters()),\n", " lr=fc.lr) # pure Adam, statute\n", " t0, killed = time.time(), None\n", " n = xtr.shape[0]\n", " for step in range(1, steps + 1):\n", " idx = torch.randint(0, n, (fc.batch,), device=dev)\n", " feats, _ = net(xtr[idx])\n", " loss = F.cross_entropy(head(feats), ytr[idx])\n", " opt.zero_grad()\n", " loss.backward()\n", " gn = torch.nn.utils.clip_grad_norm_(\n", " list(net.parameters()) + list(head.parameters()),\n", " max(loss.item(), 1.0)) # standing clip rule\n", " opt.step()\n", " if step % fc.eval_every == 0 or step == steps:\n", " reason = check_kill(loss.item(), gn.item(), net.codebook_stats())\n", " if reason:\n", " killed = reason\n", " break\n", " # rung-end gauges (also computed for killed arms — the corpse is data)\n", " net.eval()\n", " with torch.no_grad():\n", " fe, _ = net(xev)\n", " ce_bits = F.cross_entropy(head(fe), yev).item() / math.log(2)\n", " acc = (head(fe).argmax(-1) == yev).float().mean().item()\n", " # probe budget scales with class count: 250 steps underfits a\n", " # 256-way linear readout (measured: single held 3.11 vs delivered\n", " # 5.34 at 2b). Within-op comparisons survive the bias; the\n", " # held-vs-delivered gap does not, so we feed the probe properly.\n", " psteps = max(fc.probe_steps, int(2.5 * fc.bubble.n_leaves))\n", " mb = marginal_bits(net, xev, yev, fc.bubble.n_leaves,\n", " probe_steps=psteps, seed=spec.seed)\n", " cg = composition_gauges(net, xev[:1024], cluster=lev[:1024, 0])\n", " st = net.codebook_stats()\n", " row = dict(arm_id=spec.arm_id(), op=spec.op, m=spec.m, K=spec.K,\n", " d_addr=spec.d_addr, freeze=spec.freeze, seed=spec.seed,\n", " tree_hard=spec.tree_hard, steps=steps,\n", " params=net.param_count(),\n", " acc=round(acc, 4), ce_bits=round(ce_bits, 4),\n", " cum_bits=round(mb[\"cumulative_bits\"][-1], 4),\n", " delivered_bits=round(\n", " math.log2(fc.bubble.n_leaves) - ce_bits, 4),\n", " marginal_bits=json.dumps(\n", " [round(v, 4) for v in mb[\"marginal_bits\"]]),\n", " redundancy=round(cg[\"redundancy\"], 4),\n", " cancellation=round(cg[\"cancellation\"], 4),\n", " stage_snr=round(cg.get(\"stage_snr\", 0.0), 4),\n", " dev_mean=round(sum(s[\"deviation\"] for s in st) / len(st), 4),\n", " rank_mean=round(sum(s[\"eff_rank\"] for s in st) / len(st), 3),\n", " killed=killed or \"\", wall_s=round(time.time() - t0, 1))\n", " return row\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Ledger + HF push\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "_CSV_COLS = [\"rung\", \"arm_id\", \"op\", \"m\", \"K\", \"d_addr\", \"freeze\", \"seed\",\n", " \"tree_hard\", \"steps\", \"params\", \"acc\", \"ce_bits\", \"cum_bits\",\n", " \"delivered_bits\",\n", " \"marginal_bits\", \"redundancy\", \"cancellation\", \"stage_snr\",\n", " \"dev_mean\", \"rank_mean\", \"verdict\", \"reason\", \"killed\", \"wall_s\"]\n", "\n", "def ledger_append(fc: ForgeConfig, rows: List[Dict]):\n", " os.makedirs(fc.out_dir, exist_ok=True)\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " new = not os.path.exists(path)\n", " with open(path, \"a\", newline=\"\") as f:\n", " w = csv.DictWriter(f, fieldnames=_CSV_COLS, extrasaction=\"ignore\")\n", " if new:\n", " w.writeheader()\n", " w.writerows(rows)\n", "\n", "\n", "def ledger_rows(fc: ForgeConfig) -> List[Dict]:\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " if not os.path.exists(path):\n", " return []\n", " with open(path) as f:\n", " return list(csv.DictReader(f))\n", "\n", "\n", "def ledger_ids(fc: ForgeConfig) -> set:\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " if not os.path.exists(path):\n", " return set()\n", " with open(path) as f:\n", " return {r[\"arm_id\"] for r in csv.DictReader(f)}\n", "\n", "\n", "def write_sweep_md(fc: ForgeConfig, all_rows: List[Dict]):\n", " rows = sorted(all_rows, key=lambda r: -float(r[\"cum_bits\"]))\n", " lines = [\"# exp_011 ACD — sweep leaderboard\", \"\",\n", " f\"Task: nested bubbles {fc.bubble.branching} \"\n", " f\"({fc.bubble.n_leaves} leaves, \"\n", " f\"{math.log2(fc.bubble.n_leaves):.1f} bits available)\", \"\",\n", " \"| arm | rung | cum bits | marginal | acc | red | canc | dev | rank | verdict |\",\n", " \"|---|---|---|---|---|---|---|---|---|---|\"]\n", " for r in rows:\n", " lines.append(\n", " f\"| `{r['arm_id']}` | {r['rung']} | **{r['cum_bits']}** \"\n", " f\"| {r['marginal_bits']} | {r['acc']} | {r['redundancy']} \"\n", " f\"| {r['cancellation']} | {r['dev_mean']} | {r['rank_mean']} \"\n", " f\"| {r['verdict']}{(' — ' + r['reason']) if r['reason'] else ''} |\")\n", " with open(os.path.join(fc.out_dir, \"SWEEP.md\"), \"w\") as f:\n", " f.write(\"\\n\".join(lines) + \"\\n\")\n", "\n", "\n", "def hf_push(fc: ForgeConfig):\n", " if not fc.push:\n", " print(\"[push] dry run — skipped\")\n", " return\n", " from huggingface_hub import HfApi, create_repo\n", " create_repo(REPO_ID, repo_type=\"model\", exist_ok=True)\n", " api = HfApi()\n", " for name in (\"results.csv\", \"SWEEP.md\", \"verdicts.jsonl\"):\n", " p = os.path.join(fc.out_dir, name)\n", " if os.path.exists(p):\n", " api.upload_file(path_or_fileobj=p,\n", " path_in_repo=f\"{fc.exp_prefix}/{name}\",\n", " repo_id=REPO_ID, repo_type=\"model\",\n", " commit_message=f\"{fc.exp_prefix}: {name}\")\n", " print(f\"[push] {fc.exp_prefix}/ -> {REPO_ID} ✓\")\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Scheduler — successive halving with logged verdicts\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_screen(arms: List[ArmSpec], fc: ForgeConfig,\n", " rungs: Optional[List[int]] = None,\n", " protect_ops: Tuple[str, ...] = ()) -> List[Dict]:\n", " \"\"\"Run arms through the configured rungs; keep top keep_frac by\n", " cum_bits per rung (kills never advance). Returns all ledger rows.\"\"\"\n", " g = torch.Generator().manual_seed(fc.seed)\n", " bub = NestedBubbles(fc.bubble)\n", " xtr, ytr, ltr = bub.sample(fc.n_train, device=fc.device)\n", " xev, yev, lev = bub.sample(fc.n_eval, device=fc.device)\n", " data = (xtr, ytr, ltr, xev, yev, lev)\n", "\n", " alive = list(arms)\n", " if not alive:\n", " prior = ledger_rows(fc)\n", " print(f\"[screen] nothing queued — ledger already holds \"\n", " f\"{len(prior)} rows in {fc.out_dir}/results.csv; \"\n", " f\"call report(fc) to view standings.\")\n", " return prior\n", " all_rows: List[Dict] = []\n", " vpath = os.path.join(fc.out_dir, \"verdicts.jsonl\")\n", " os.makedirs(fc.out_dir, exist_ok=True)\n", " for ri, (tier, steps, keep) in enumerate(fc.rungs):\n", " if rungs is not None and ri not in rungs:\n", " continue\n", " if tier == \"L\":\n", " try:\n", " run_arm_L = _ns(\"run_arm_L\", \"acd_lm_adapter\")\n", " tier_l_overrides = _ns(\"tier_l_overrides\",\n", " \"acd_lm_adapter\")\n", " MarkovByteStream = _ns(\"MarkovByteStream\",\n", " \"acd_lm_adapter\")\n", " except Exception as e:\n", " raise RuntimeError(\n", " \"Tier-L needs acd_lm_adapter in the namespace — \"\n", " \"paste it (after the aleph-lm cells) first.\") from e\n", " if not hasattr(fc, \"_lm_stream\"):\n", " if fc.lm_corpus == \"synthetic\":\n", " fc._lm_stream = MarkovByteStream(seed=fc.seed)\n", " else:\n", " TrigramStream = _ns(\"TrigramStream\", \"aleph_lm\")\n", " fc._lm_stream = TrigramStream(\n", " fc.lm_corpus, max_corpus_bytes=100_000_000,\n", " seed=fc.seed)\n", " lm_over = fc.lm or tier_l_overrides()\n", " run_L = lambda sp, st: run_arm_L(\n", " sp, st, fc._lm_stream, lm_over, fc.device,\n", " probe_steps=max(fc.probe_steps, 640),\n", " eval_every=fc.eval_every)\n", " cached = {r[\"arm_id\"]: r for r in ledger_rows(fc)\n", " if r.get(\"rung\") == str(ri)}\n", " n_hit = sum(1 for s in alive if s.arm_id() in cached)\n", " print(f\"\\n[rung {ri}] tier={tier} steps={steps} \"\n", " f\"arms={len(alive)} (cached {n_hit}, \"\n", " f\"fresh {len(alive) - n_hit})\")\n", " rows, fresh = [], []\n", " for k, spec in enumerate(alive):\n", " aid = spec.arm_id()\n", " if aid in cached: # RESUME: reuse the ledger row\n", " rows.append(cached[aid])\n", " print(f\" [{k+1}/{len(alive)}] {aid:34s} cached \"\n", " f\"(bits {cached[aid]['cum_bits']})\")\n", " continue\n", " row = (run_arm_P(spec, steps, fc, data) if tier == \"P\"\n", " else run_L(spec, steps))\n", " row[\"rung\"] = ri\n", " rows.append(row)\n", " fresh.append(row)\n", " print(f\" [{k+1}/{len(alive)}] {row['arm_id']:34s} \"\n", " f\"held {row['cum_bits']:.2f} ce {row['ce_bits']:.3f} \"\n", " f\"dlv {row['delivered_bits']:.2f} \"\n", " f\"{row['marginal_bits']} \"\n", " f\"acc {row['acc']:.3f} red {row['redundancy']:.2f}\"\n", " f\"{' KILLED: ' + row['killed'] if row['killed'] else ''}\")\n", " # verdicts\n", " survivors = [r for r in rows if not r[\"killed\"]]\n", " survivors.sort(key=lambda r: -float(r[\"cum_bits\"]))\n", " n_keep = max(1, int(len(survivors) * keep))\n", " promoted = {r[\"arm_id\"] for r in survivors[:n_keep]}\n", " # science over leaderboard: controls ride every rung so the\n", " # divergence gate is actually tested at full training depth\n", " shielded = {r[\"arm_id\"] for r in survivors\n", " if r[\"op\"] in protect_ops}\n", " promoted |= shielded\n", " with open(vpath, \"a\") as vf:\n", " for r in rows:\n", " if r[\"killed\"]:\n", " r[\"verdict\"], r[\"reason\"] = \"KILL\", r[\"killed\"]\n", " elif r[\"arm_id\"] in promoted:\n", " why = (\"protected control\"\n", " if r[\"op\"] in protect_ops\n", " and r[\"arm_id\"] not in\n", " {s[\"arm_id\"] for s in survivors[:n_keep]}\n", " else f\"top {n_keep}/{len(survivors)} by cum_bits\")\n", " r[\"verdict\"], r[\"reason\"] = \"PROMOTE\", why\n", " else:\n", " r[\"verdict\"], r[\"reason\"] = \"PARK\", \"below keep line\"\n", " if any(r is fr for fr in fresh):\n", " vf.write(json.dumps(r) + \"\\n\")\n", " ledger_append(fc, fresh)\n", " all_rows += rows\n", " write_sweep_md(fc, all_rows)\n", " hf_push(fc)\n", " by_id = {a.arm_id(): a for a in alive}\n", " alive = [by_id[i] for i in promoted if i in by_id]\n", " if not alive:\n", " print(\"[screen] no survivors — stopping\")\n", " break\n", " return all_rows\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Activation — dual-mode (script main / notebook cell)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def phase2_screen(push: bool = True):\n", " \"\"\"The Phase-2 mass screen: full operator grid, rung0+rung1.\"\"\"\n", " fc = ForgeConfig(push=push)\n", " ops = [\"sum\", \"gate\", \"res\", \"prod\", \"tree\", \"cross\", \"anneal\"]\n", " arms = generate_arms(ops, ms=[2, 3, 4], freezes=[\"free\", \"spread\"],\n", " seeds=[1234]) # resume-safe: cache skips reruns\n", " print(f\"[forge] {len(arms)} arms queued (incl. controls)\")\n", " return run_screen(arms, fc, rungs=[0, 1])\n", "\n", "\n", "def phase2b_screen(push: bool = True):\n", " \"\"\"Phase 2b: divergence hunt with headroom. 256-leaf task (8 bits),\n", " m up to 8, two seeds, SUM + SINGLE protected to full training depth.\n", " Own ledger (exp011b/) — the task changed, so rows must not mix.\"\"\"\n", " fc = ForgeConfig(out_dir=\"exp011b\", push=push,\n", " n_train=16384, n_eval=8192, batch=1024,\n", " bubble=BubbleConfig(d_data=32, branching=(4, 4, 4, 4),\n", " sep0=6.0, sep_decay=0.5,\n", " noise=0.3, seed=97))\n", " ops = [\"sum\", \"gate\", \"res\", \"prod\", \"cross\"] # anneal parked (rung-0\n", " # verdict: clone stages); tree awaits the budget-accounting decision\n", " arms = generate_arms(ops, ms=[2, 4, 6, 8], freezes=[\"free\", \"spread\"],\n", " seeds=[1234, 5678])\n", " print(f\"[forge] {len(arms)} arms queued (sum+single protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", "\n", "\n", "def phase3_screen(push: bool = True):\n", " \"\"\"Phase 3: Tier-P finalists onto the 6.75M LM (WikiText-103).\n", " prod/res at their knees + sum control + singles, byte head first\n", " (bank-free); hybrid/apmix arms follow once the byte read is banked.\n", " ce_bits column = bits/byte; delivered_bits = 8 - bpb.\"\"\"\n", " tier_l_overrides = _ns(\"tier_l_overrides\", \"acd_lm_adapter\")\n", " fc = ForgeConfig(out_dir=\"exp011L\", push=push,\n", " rungs=((\"L\", 1000, 0.5), (\"L\", 5000, 1.0)),\n", " lm=tier_l_overrides())\n", " arms = generate_arms([\"prod\", \"res\", \"sum\"], ms=[8],\n", " freezes=[\"free\"], seeds=[1234, 5678])\n", " arms += generate_arms([\"res\"], ms=[16], freezes=[\"free\"],\n", " seeds=[1234, 5678],\n", " existing_ids={a.arm_id() for a in arms})\n", " print(f\"[forge] {len(arms)} Tier-L arms queued (sum+single protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", "\n", "\n", "def report(fc: Optional[ForgeConfig] = None, top: int = 15):\n", " \"\"\"Print standings from the ledger without running anything.\"\"\"\n", " fc = fc or ForgeConfig(push=False)\n", " rows = ledger_rows(fc)\n", " if not rows:\n", " print(f\"[report] no ledger at {fc.out_dir}/results.csv\")\n", " return rows\n", " rows.sort(key=lambda r: (-int(r[\"rung\"]), -float(r[\"cum_bits\"])))\n", " print(f\"[report] {len(rows)} rows — top {top}:\")\n", " for r in rows[:top]:\n", " k = \" KILLED:\" + r[\"killed\"] if r[\"killed\"] else \"\"\n", " print(f\" r{r['rung']} {r['arm_id']:36s} bits {r['cum_bits']:>7s} \"\n", " f\"acc {r['acc']:>6s} red {r['redundancy']:>6s} \"\n", " f\"{r['verdict']}{k}\")\n", " return rows\n", "\n", "\n", "def _smoke():\n", " # Self-cleaning: Colab working dirs PERSIST across sessions, and a\n", " # stale smoke ledger turns fresh-arm tests into cache hits. The\n", " # smoke always starts from a swept room; resume behavior is tested\n", " # WITHIN the smoke (first pass trains, second pass must cache).\n", " import shutil\n", " for _d in (\"exp011_smoke\", \"exp011_smoke_prot\"):\n", " shutil.rmtree(_d, ignore_errors=True)\n", " fc = ForgeConfig(push=False, n_train=1536, n_eval=768, batch=256,\n", " probe_steps=120, eval_every=25,\n", " rungs=((\"P\", 60, 0.5), (\"P\", 120, 1.0)),\n", " out_dir=\"exp011_smoke\")\n", " arms = generate_arms([\"sum\", \"res\", \"prod\"], ms=[3], freezes=[\"free\"],\n", " seeds=[1234])\n", " ids = [a.arm_id() for a in arms]\n", " assert len(ids) == len(set(ids)), \"id collision\"\n", " assert any(a.op == \"single\" for a in arms), \"budget twin missing\"\n", " print(f\" ✓ generator: {len(arms)} arms \"\n", " f\"({[a.op for a in arms]})\")\n", " # kill-rule unit checks\n", " assert check_kill(float(\"nan\"), 1.0, []) == \"NaN/inf loss\"\n", " assert check_kill(1.0, 5e3, []).startswith(\"grad blowup\")\n", " assert check_kill(1.0, 1.0, [{\"eff_rank\": 1.2}]).startswith(\"rank collapse\")\n", " assert check_kill(1.0, 1.0, [{\"eff_rank\": 3.8}]) is None\n", " print(\" ✓ kill rules fire correctly\")\n", " rows = run_screen(arms, fc, rungs=[0, 1])\n", " assert os.path.exists(os.path.join(fc.out_dir, \"results.csv\"))\n", " assert os.path.exists(os.path.join(fc.out_dir, \"SWEEP.md\"))\n", " assert os.path.exists(os.path.join(fc.out_dir, \"verdicts.jsonl\"))\n", " assert all(r[\"verdict\"] in (\"PROMOTE\", \"PARK\", \"KILL\") for r in rows)\n", " # RESUME test: re-run same arms -> all cached, nothing trains;\n", " # add one new op -> only it trains, union re-ranked\n", " n_ledger_before = len(ledger_rows(fc))\n", " t0 = time.time()\n", " rows2 = run_screen(arms, fc, rungs=[0, 1])\n", " assert time.time() - t0 < 20, \"resume re-trained instead of caching\"\n", " assert len(ledger_rows(fc)) == n_ledger_before, \"cache appended rows\"\n", " r0 = [r for r in rows2 if str(r[\"rung\"]) == \"0\"]\n", " assert r0 and all(isinstance(r[\"cum_bits\"], str) for r in r0), \\\n", " \"rung0 rows were re-trained (fresh rows are floats)\"\n", " arms3 = generate_arms([\"sum\", \"res\", \"prod\", \"gate\"], ms=[3],\n", " freezes=[\"free\"], seeds=[1234])\n", " rows3 = run_screen(arms3, fc, rungs=[0])\n", " fresh_gate = [r for r in rows3 if r[\"op\"] == \"gate\"]\n", " assert len(fresh_gate) == 1 and isinstance(fresh_gate[0][\"cum_bits\"], float)\n", " assert isinstance(fresh_gate[0].get(\"delivered_bits\"), float), \\\n", " \"delivered_bits missing from fresh rows\"\n", " print(\" ✓ resume: full-cache fast path + mixed cache/fresh both work\")\n", " # protection: sum rides to rung1 despite bottom rank\n", " fcp = ForgeConfig(push=False, n_train=1024, n_eval=512, batch=256,\n", " probe_steps=80, eval_every=25,\n", " rungs=((\"P\", 40, 0.34), (\"P\", 60, 1.0)),\n", " out_dir=\"exp011_smoke_prot\")\n", " armsp = generate_arms([\"sum\", \"res\", \"prod\"], ms=[3], freezes=[\"free\"],\n", " seeds=[1234])\n", " rowsp = run_screen(armsp, fcp, rungs=[0, 1], protect_ops=(\"sum\",))\n", " r1_ops = {r[\"op\"] for r in rowsp if str(r[\"rung\"]) == \"1\"}\n", " assert \"sum\" in r1_ops, f\"protected sum culled: rung1 ops {r1_ops}\"\n", " prot = [r for r in rowsp if r[\"op\"] == \"sum\" and str(r[\"rung\"]) == \"0\"]\n", " assert any(\"protected\" in r[\"reason\"] or \"top\" in r[\"reason\"]\n", " for r in prot)\n", " print(\" ✓ protected controls ride the keep-line\")\n", " # Tier-L path (synthetic stream; skipped if adapter not pasted yet)\n", " try:\n", " smoke_overrides = _ns(\"smoke_overrides\", \"acd_lm_adapter\")\n", " import shutil as _sh\n", " _sh.rmtree(\"exp011_smoke_L\", ignore_errors=True)\n", " fcl = ForgeConfig(out_dir=\"exp011_smoke_L\", push=False,\n", " rungs=((\"L\", 25, 1.0),), probe_steps=150,\n", " eval_every=20, lm=smoke_overrides(),\n", " lm_corpus=\"synthetic\")\n", " armsl = generate_arms([\"prod\"], ms=[2], freezes=[\"free\"],\n", " seeds=[1234])\n", " rowsl = run_screen(armsl, fcl, rungs=[0])\n", " fr = [r for r in rowsl if isinstance(r[\"ce_bits\"], float)]\n", " assert fr and all(r[\"ce_bits\"] < 8.0 for r in fr)\n", " assert all(isinstance(r[\"delivered_bits\"], float) for r in fr)\n", " print(\" ✓ Tier-L rung: bpb finite, delivered logged, \"\n", " \"ledger schema intact\")\n", " except Exception:\n", " print(\" - Tier-L smoke skipped (acd_lm_adapter not in namespace)\")\n", " # empty-queue guard returns ledger instead of []\n", " empty = run_screen([], fc, rungs=[0])\n", " assert len(empty) > 0, \"empty guard returned nothing\"\n", " print(\" ✓ empty-queue guard returns ledger standings\")\n", " print(\"acd_forge smoke: ALL GREEN — next: phase2_screen()\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " # Notebook cells execute as __main__, so the smoke fires on paste too —\n", " # deliberate: pasting a cell IS the verification step in the Colab flow\n", " # (shared namespace, paste order structures -> probe -> forge).\n", " # Heavy entry points (phase2_screen) are never wired here; call them.\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "lUEqlSR6gLeF", "outputId": "c03c0008-9b0d-4188-bb6b-330b2ab5fbf7" }, "execution_count": 3, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ generator: 4 arms (['sum', 'res', 'prod', 'single'])\n", " ✓ kill rules fire correctly\n", "\n", "[rung 0] tier=P steps=60 arms=4 (cached 0, fresh 4)\n", " [1/4] sum_m3_K21_free_s1234_7faffe held 3.68 ce 4.499 dlv 1.50 [2.9094, 0.2359, 0.5361] acc 0.111 red 0.23\n", " [2/4] res_m3_K21_free_s1234_1925f3 held 4.15 ce 3.174 dlv 2.83 [3.3556, 0.216, 0.5828] acc 0.310 red 0.08\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 held 4.13 ce 3.063 dlv 2.94 [3.0398, 0.5619, 0.5312] acc 0.253 red 0.05\n", " [4/4] single_m1_K64_free_s1234_194318 held 2.95 ce 4.330 dlv 1.67 [2.9487] acc 0.185 red 0.00\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=120 arms=2 (cached 0, fresh 2)\n", " [1/2] prod_m3_K21_free_s1234_c5d5b7 held 4.36 ce 1.970 dlv 4.03 [3.2752, 0.5116, 0.5715] acc 0.453 red 0.05\n", " [2/2] res_m3_K21_free_s1234_1925f3 held 4.41 ce 1.938 dlv 4.06 [3.4941, 0.2551, 0.6582] acc 0.466 red 0.06\n", "[push] dry run — skipped\n", "\n", "[rung 0] tier=P steps=60 arms=4 (cached 4, fresh 0)\n", " [1/4] sum_m3_K21_free_s1234_7faffe cached (bits 3.6814)\n", " [2/4] res_m3_K21_free_s1234_1925f3 cached (bits 4.1544)\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.1329)\n", " [4/4] single_m1_K64_free_s1234_194318 cached (bits 2.9487)\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=120 arms=2 (cached 2, fresh 0)\n", " [1/2] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.3582)\n", " [2/2] res_m3_K21_free_s1234_1925f3 cached (bits 4.4074)\n", "[push] dry run — skipped\n", "\n", "[rung 0] tier=P steps=60 arms=5 (cached 4, fresh 1)\n", " [1/5] sum_m3_K21_free_s1234_7faffe cached (bits 3.6814)\n", " [2/5] res_m3_K21_free_s1234_1925f3 cached (bits 4.1544)\n", " [3/5] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.1329)\n", " [4/5] gate_m3_K21_free_s1234_0d3f58 held 3.54 ce 4.379 dlv 1.62 [2.7871, 0.3954, 0.3608] acc 0.134 red 0.27\n", " [5/5] single_m1_K64_free_s1234_194318 cached (bits 2.9487)\n", "[push] dry run — skipped\n", " ✓ resume: full-cache fast path + mixed cache/fresh both work\n", "\n", "[rung 0] tier=P steps=40 arms=4 (cached 0, fresh 4)\n", " [1/4] sum_m3_K21_free_s1234_7faffe held 3.46 ce 5.407 dlv 0.59 [2.4185, 0.8562, 0.1877] acc 0.123 red 0.18\n", " [2/4] res_m3_K21_free_s1234_1925f3 held 3.72 ce 4.209 dlv 1.79 [2.8238, 0.6729, 0.2209] acc 0.203 red 0.11\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 held 3.67 ce 4.186 dlv 1.81 [2.6164, 0.6287, 0.4231] acc 0.221 red 0.07\n", " [4/4] single_m1_K64_free_s1234_194318 held 2.47 ce 5.279 dlv 0.72 [2.4703] acc 0.166 red 0.00\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=60 arms=2 (cached 0, fresh 2)\n", " [1/2] res_m3_K21_free_s1234_1925f3 held 3.80 ce 3.152 dlv 2.85 [2.9599, 0.6163, 0.2213] acc 0.268 red 0.09\n", " [2/2] sum_m3_K21_free_s1234_7faffe held 3.38 ce 4.597 dlv 1.40 [2.5272, 0.7302, 0.1235] acc 0.135 red 0.22\n", "[push] dry run — skipped\n", " ✓ protected controls ride the keep-line\n", " - Tier-L smoke skipped (acd_lm_adapter not in namespace)\n", "[screen] nothing queued — ledger already holds 7 rows in exp011_smoke/results.csv; call report(fc) to view standings.\n", " ✓ empty-queue guard returns ledger standings\n", "acd_forge smoke: ALL GREEN — next: phase2_screen()\n" ] } ] }, { "cell_type": "code", "source": [ "phase2_screen()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "01vJ8hoaoIpl", "outputId": "31f0c3b4-9f6a-49b0-b169-acd6abec62cb" }, "execution_count": 4, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[forge] 50 arms queued (incl. controls)\n", "\n", "[rung 0] tier=P steps=200 arms=50 (cached 0, fresh 50)\n", " [1/50] sum_m2_K32_free_s1234_6e47da held 4.30 ce 2.068 dlv 3.93 [3.6925, 0.6054] acc 0.446 red 0.17\n", " [2/50] sum_m2_K32_spread_s1234_6f48c1 held 4.41 ce 2.057 dlv 3.94 [3.8262, 0.5792] acc 0.508 red 0.10\n", " [3/50] sum_m3_K21_free_s1234_7faffe held 4.34 ce 2.114 dlv 3.89 [3.6162, 0.5073, 0.2135] acc 0.364 red 0.22\n", " [4/50] sum_m3_K21_spread_s1234_5cce31 held 4.59 ce 2.159 dlv 3.84 [3.6675, 0.6584, 0.2597] acc 0.427 red 0.17\n", " [5/50] sum_m4_K16_free_s1234_cf35b9 held 4.40 ce 2.208 dlv 3.79 [3.3976, 0.6311, 0.2206, 0.1477] acc 0.366 red 0.33\n", " [6/50] sum_m4_K16_spread_s1234_771ef0 held 4.57 ce 2.227 dlv 3.77 [3.3456, 0.7253, 0.2353, 0.2658] acc 0.378 red 0.24\n", " [7/50] gate_m2_K32_free_s1234_62d38f held 4.21 ce 2.039 dlv 3.96 [3.6712, 0.5384] acc 0.408 red 0.15\n", " [8/50] gate_m2_K32_spread_s1234_c08006 held 4.27 ce 1.994 dlv 4.01 [3.861, 0.4049] acc 0.469 red 0.14\n", " [9/50] gate_m3_K21_free_s1234_0d3f58 held 4.31 ce 2.342 dlv 3.66 [3.2963, 0.8447, 0.1673] acc 0.322 red 0.25\n", " [10/50] gate_m3_K21_spread_s1234_578051 held 4.46 ce 2.063 dlv 3.94 [3.3476, 0.819, 0.2928] acc 0.430 red 0.18\n", " [11/50] gate_m4_K16_free_s1234_b3217d held 4.38 ce 2.188 dlv 3.81 [3.0458, 0.7406, 0.4888, 0.1088] acc 0.344 red 0.28\n", " [12/50] gate_m4_K16_spread_s1234_391fa6 held 4.61 ce 2.254 dlv 3.75 [3.4253, 0.5177, 0.4491, 0.2148] acc 0.402 red 0.32\n", " [13/50] res_m2_K32_free_s1234_77f47d held 4.60 ce 1.510 dlv 4.49 [3.6831, 0.9143] acc 0.604 red 0.03\n", " [14/50] res_m2_K32_spread_s1234_6badfb held 4.70 ce 1.290 dlv 4.71 [3.8575, 0.8423] acc 0.685 red 0.05\n", " [15/50] res_m3_K21_free_s1234_1925f3 held 5.00 ce 1.105 dlv 4.89 [3.8044, 0.7096, 0.4822] acc 0.729 red 0.07\n", " [16/50] res_m3_K21_spread_s1234_7fd02e held 4.96 ce 1.062 dlv 4.94 [3.7923, 0.8558, 0.315] acc 0.744 red 0.07\n", " [17/50] res_m4_K16_free_s1234_4c2e4d held 5.18 ce 0.931 dlv 5.07 [3.7297, 0.8697, 0.3736, 0.2048] acc 0.773 red 0.07\n", " [18/50] res_m4_K16_spread_s1234_0d417f held 5.15 ce 0.895 dlv 5.10 [3.4718, 0.9779, 0.4586, 0.2448] acc 0.784 red 0.06\n", " [19/50] prod_m2_K32_free_s1234_c2fd6a held 4.63 ce 1.362 dlv 4.64 [3.5051, 1.123] acc 0.658 red 0.04\n", " [20/50] prod_m2_K32_spread_s1234_2a0e45 held 4.63 ce 1.283 dlv 4.72 [3.7031, 0.9276] acc 0.691 red 0.07\n", " [21/50] prod_m3_K21_free_s1234_c5d5b7 held 4.99 ce 1.130 dlv 4.87 [3.5967, 1.0109, 0.3805] acc 0.743 red 0.07\n", " [22/50] prod_m3_K21_spread_s1234_a0d133 held 5.02 ce 1.027 dlv 4.97 [3.6619, 1.0069, 0.3529] acc 0.765 red 0.05\n", " [23/50] prod_m4_K16_free_s1234_be714a held 5.24 ce 0.865 dlv 5.13 [3.6226, 0.9476, 0.4001, 0.2708] acc 0.809 red 0.07\n", " [24/50] prod_m4_K16_spread_s1234_77144d held 5.24 ce 0.788 dlv 5.21 [3.4752, 1.1586, 0.4234, 0.1859] acc 0.820 red 0.08\n", " [25/50] tree_m2_K3_free_s1234_7152c6 held 3.99 ce 2.077 dlv 3.92 [3.4395, 0.5526] acc 0.396 red 0.02\n", " [26/50] tree_m2_K3_hard_free_s1234_42aca9 held 3.84 ce 2.275 dlv 3.73 [3.2426, 0.5975] acc 0.335 red 0.01\n", " [27/50] tree_m2_K3_spread_s1234_667729 held 3.96 ce 2.098 dlv 3.90 [3.3019, 0.6532] acc 0.343 red 0.15\n", " [28/50] tree_m2_K3_hard_spread_s1234_8dff78 held 3.84 ce 2.220 dlv 3.78 [3.2043, 0.6343] acc 0.316 red 0.15\n", " [29/50] tree_m3_K3_free_s1234_da1e88 held 3.99 ce 2.077 dlv 3.92 [3.4395, 0.5526] acc 0.396 red 0.02\n", " [30/50] tree_m3_K3_hard_free_s1234_709495 held 3.84 ce 2.275 dlv 3.73 [3.2426, 0.5975] acc 0.335 red 0.01\n", " [31/50] tree_m3_K3_spread_s1234_574e19 held 3.96 ce 2.098 dlv 3.90 [3.3019, 0.6532] acc 0.343 red 0.15\n", " [32/50] tree_m3_K3_hard_spread_s1234_a37231 held 3.84 ce 2.220 dlv 3.78 [3.2043, 0.6343] acc 0.316 red 0.15\n", " [33/50] tree_m4_K3_free_s1234_5884d1 held 3.99 ce 2.077 dlv 3.92 [3.4395, 0.5526] acc 0.396 red 0.02\n", " [34/50] tree_m4_K3_hard_free_s1234_d48cf2 held 3.84 ce 2.275 dlv 3.73 [3.2426, 0.5975] acc 0.335 red 0.01\n", " [35/50] tree_m4_K3_spread_s1234_5d0ded held 3.96 ce 2.098 dlv 3.90 [3.3019, 0.6532] acc 0.343 red 0.15\n", " [36/50] tree_m4_K3_hard_spread_s1234_7f8025 held 3.84 ce 2.220 dlv 3.78 [3.2043, 0.6343] acc 0.316 red 0.15\n", " [37/50] cross_m2_K32_free_s1234_843ff1 held 4.65 ce 1.285 dlv 4.72 [3.7766, 0.8693] acc 0.650 red 0.05\n", " [38/50] cross_m2_K32_spread_s1234_c82ea9 held 4.65 ce 1.151 dlv 4.85 [3.6905, 0.9596] acc 0.702 red 0.04\n", " [39/50] cross_m3_K21_free_s1234_ae1dcc held 4.95 ce 0.987 dlv 5.01 [3.7786, 0.7698, 0.4012] acc 0.752 red 0.09\n", " [40/50] cross_m3_K21_spread_s1234_e95e85 held 5.01 ce 0.911 dlv 5.09 [3.6384, 0.923, 0.444] acc 0.763 red 0.05\n", " [41/50] cross_m4_K16_free_s1234_f28e7d held 5.22 ce 0.717 dlv 5.28 [3.4413, 1.0572, 0.4574, 0.2683] acc 0.819 red 0.09\n", " [42/50] cross_m4_K16_spread_s1234_d48dff held 5.18 ce 0.769 dlv 5.23 [3.4292, 1.0595, 0.4405, 0.2475] acc 0.810 red 0.08\n", " [43/50] anneal_m2_K64_free_s1234_3cd58b held 4.00 ce 1.881 dlv 4.12 [3.6609, 0.3387] acc 0.466 red 0.95\n", " [44/50] anneal_m2_K64_spread_s1234_685a4a held 3.98 ce 1.791 dlv 4.21 [3.576, 0.4062] acc 0.482 red 0.93\n", " [45/50] anneal_m3_K64_free_s1234_27a360 held 4.18 ce 1.695 dlv 4.30 [3.6599, 0.4232, 0.0948] acc 0.507 red 0.92\n", " [46/50] anneal_m3_K64_spread_s1234_2320dd held 4.09 ce 1.769 dlv 4.23 [3.5418, 0.4811, 0.0661] acc 0.502 red 0.91\n", " [47/50] anneal_m4_K64_free_s1234_238ae3 held 4.14 ce 1.781 dlv 4.22 [3.6047, 0.4819, 0.0304, 0.0272] acc 0.506 red 0.90\n", " [48/50] anneal_m4_K64_spread_s1234_dd4475 held 4.13 ce 1.743 dlv 4.26 [3.6392, 0.4584, 0.0201, 0.0118] acc 0.516 red 0.90\n", " [49/50] single_m1_K64_free_s1234_194318 held 3.83 ce 1.954 dlv 4.05 [3.8274] acc 0.431 red 0.00\n", " [50/50] single_m1_K64_spread_s1234_5dc570 held 3.73 ce 1.855 dlv 4.14 [3.7269] acc 0.503 red 0.00\n", "[push] exp011/ -> AbstractPhil/geolip-aleph-differentiation ✓\n", "\n", "[rung 1] tier=P steps=1000 arms=16 (cached 0, fresh 16)\n", " [1/16] prod_m4_K16_spread_s1234_77144d held 5.36 ce 0.466 dlv 5.53 [3.2781, 1.3449, 0.4821, 0.257] acc 0.892 red 0.09\n", " [2/16] cross_m2_K32_free_s1234_843ff1 held 4.83 ce 0.793 dlv 5.21 [3.7992, 1.0295] acc 0.802 red 0.07\n", " [3/16] cross_m4_K16_spread_s1234_d48dff held 5.33 ce 0.560 dlv 5.44 [3.4055, 1.1316, 0.5043, 0.2852] acc 0.883 red 0.08\n", " [4/16] cross_m3_K21_spread_s1234_e95e85 held 5.18 ce 0.595 dlv 5.40 [3.5683, 1.083, 0.5286] acc 0.860 red 0.05\n", " [5/16] prod_m3_K21_free_s1234_c5d5b7 held 5.18 ce 0.591 dlv 5.41 [3.3104, 1.2918, 0.5824] acc 0.858 red 0.07\n", " [6/16] cross_m3_K21_free_s1234_ae1dcc held 5.18 ce 0.612 dlv 5.39 [3.7605, 0.9527, 0.466] acc 0.863 red 0.09\n", " [7/16] res_m3_K21_free_s1234_1925f3 held 5.26 ce 0.625 dlv 5.38 [3.839, 0.8904, 0.5285] acc 0.858 red 0.08\n", " [8/16] res_m4_K16_spread_s1234_0d417f held 5.35 ce 0.511 dlv 5.49 [3.4717, 1.1155, 0.4902, 0.2704] acc 0.885 red 0.08\n", " [9/16] prod_m4_K16_free_s1234_be714a held 5.40 ce 0.470 dlv 5.53 [3.4495, 1.1775, 0.502, 0.2737] acc 0.895 red 0.08\n", " [10/16] res_m3_K21_spread_s1234_7fd02e held 5.20 ce 0.607 dlv 5.39 [3.6723, 1.0794, 0.4444] acc 0.864 red 0.07\n", " [11/16] cross_m4_K16_free_s1234_f28e7d held 5.39 ce 0.513 dlv 5.49 [3.5096, 1.0509, 0.4917, 0.3363] acc 0.890 red 0.10\n", " [12/16] cross_m2_K32_spread_s1234_c82ea9 held 4.84 ce 0.758 dlv 5.24 [3.6205, 1.2168] acc 0.815 red 0.04\n", " [13/16] prod_m3_K21_spread_s1234_a0d133 held 5.19 ce 0.551 dlv 5.45 [3.4259, 1.2627, 0.4989] acc 0.866 red 0.06\n", " [14/16] res_m2_K32_spread_s1234_6badfb held 4.89 ce 0.757 dlv 5.24 [3.9044, 0.9821] acc 0.813 red 0.06\n", " [15/16] res_m4_K16_free_s1234_4c2e4d held 5.41 ce 0.503 dlv 5.50 [3.6992, 1.0163, 0.4287, 0.263] acc 0.891 red 0.08\n", " [16/16] prod_m2_K32_spread_s1234_2a0e45 held 4.83 ce 0.743 dlv 5.26 [3.4711, 1.3614] acc 0.815 red 0.09\n", "[push] exp011/ -> AbstractPhil/geolip-aleph-differentiation ✓\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "[{'arm_id': 'sum_m2_K32_free_s1234_6e47da',\n", " 'op': 'sum',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8834,\n", " 'acc': 0.446,\n", " 'ce_bits': 2.0676,\n", " 'cum_bits': 4.2979,\n", " 'delivered_bits': 3.9324,\n", " 'marginal_bits': '[3.6925, 0.6054]',\n", " 'redundancy': 0.1713,\n", " 'cancellation': 0.5095,\n", " 'stage_snr': 0.4383,\n", " 'dev_mean': -0.1556,\n", " 'rank_mean': 3.53,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m2_K32_spread_s1234_6f48c1',\n", " 'op': 'sum',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8834,\n", " 'acc': 0.5076,\n", " 'ce_bits': 2.0568,\n", " 'cum_bits': 4.4054,\n", " 'delivered_bits': 3.9432,\n", " 'marginal_bits': '[3.8262, 0.5792]',\n", " 'redundancy': 0.1034,\n", " 'cancellation': 0.4687,\n", " 'stage_snr': 0.3486,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m3_K21_free_s1234_7faffe',\n", " 'op': 'sum',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8895,\n", " 'acc': 0.3645,\n", " 'ce_bits': 2.1135,\n", " 'cum_bits': 4.3371,\n", " 'delivered_bits': 3.8865,\n", " 'marginal_bits': '[3.6162, 0.5073, 0.2135]',\n", " 'redundancy': 0.2157,\n", " 'cancellation': 0.6854,\n", " 'stage_snr': 0.631,\n", " 'dev_mean': -0.1346,\n", " 'rank_mean': 3.558,\n", " 'killed': '',\n", " 'wall_s': 0.9,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m3_K21_spread_s1234_5cce31',\n", " 'op': 'sum',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8895,\n", " 'acc': 0.4272,\n", " 'ce_bits': 2.1586,\n", " 'cum_bits': 4.5856,\n", " 'delivered_bits': 3.8414,\n", " 'marginal_bits': '[3.6675, 0.6584, 0.2597]',\n", " 'redundancy': 0.1724,\n", " 'cancellation': 0.6604,\n", " 'stage_snr': 0.4662,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.8,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m4_K16_free_s1234_cf35b9',\n", " 'op': 'sum',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9220,\n", " 'acc': 0.3665,\n", " 'ce_bits': 2.2085,\n", " 'cum_bits': 4.3969,\n", " 'delivered_bits': 3.7915,\n", " 'marginal_bits': '[3.3976, 0.6311, 0.2206, 0.1477]',\n", " 'redundancy': 0.325,\n", " 'cancellation': 0.7142,\n", " 'stage_snr': 0.6183,\n", " 'dev_mean': -0.0867,\n", " 'rank_mean': 3.653,\n", " 'killed': '',\n", " 'wall_s': 1.1,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m4_K16_spread_s1234_771ef0',\n", " 'op': 'sum',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9220,\n", " 'acc': 0.3777,\n", " 'ce_bits': 2.2272,\n", " 'cum_bits': 4.5721,\n", " 'delivered_bits': 3.7728,\n", " 'marginal_bits': '[3.3456, 0.7253, 0.2353, 0.2658]',\n", " 'redundancy': 0.2415,\n", " 'cancellation': 0.8515,\n", " 'stage_snr': 0.5748,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 1.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'gate_m2_K32_free_s1234_62d38f',\n", " 'op': 'gate',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8978,\n", " 'acc': 0.4084,\n", " 'ce_bits': 2.0395,\n", " 'cum_bits': 4.2096,\n", " 'delivered_bits': 3.9605,\n", " 'marginal_bits': '[3.6712, 0.5384]',\n", " 'redundancy': 0.1503,\n", " 'cancellation': 0.4751,\n", " 'stage_snr': 0.552,\n", " 'dev_mean': -0.1085,\n", " 'rank_mean': 3.602,\n", " 'killed': '',\n", " 'wall_s': 0.8,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'gate_m2_K32_spread_s1234_c08006',\n", " 'op': 'gate',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8978,\n", " 'acc': 0.4685,\n", " 'ce_bits': 1.9936,\n", " 'cum_bits': 4.2659,\n", " 'delivered_bits': 4.0064,\n", " 'marginal_bits': '[3.861, 0.4049]',\n", " 'redundancy': 0.1446,\n", " 'cancellation': 0.4382,\n", " 'stage_snr': 0.5137,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 0.7,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'gate_m3_K21_free_s1234_0d3f58',\n", " 'op': 'gate',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9053,\n", " 'acc': 0.3215,\n", " 'ce_bits': 2.3425,\n", " 'cum_bits': 4.3082,\n", " 'delivered_bits': 3.6575,\n", " 'marginal_bits': '[3.2963, 0.8447, 0.1673]',\n", " 'redundancy': 0.2471,\n", " 'cancellation': 0.6441,\n", " 'stage_snr': 0.7021,\n", " 'dev_mean': -0.077,\n", " 'rank_mean': 3.698,\n", " 'killed': '',\n", " 'wall_s': 1.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'gate_m3_K21_spread_s1234_578051',\n", " 'op': 'gate',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9053,\n", " 'acc': 0.4304,\n", " 'ce_bits': 2.0628,\n", " 'cum_bits': 4.4594,\n", " 'delivered_bits': 3.9372,\n", " 'marginal_bits': '[3.3476, 0.819, 0.2928]',\n", " 'redundancy': 0.1758,\n", " 'cancellation': 0.7084,\n", " 'stage_snr': 0.6652,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.9,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'gate_m4_K16_free_s1234_b3217d',\n", " 'op': 'gate',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9396,\n", " 'acc': 0.344,\n", " 'ce_bits': 2.1878,\n", " 'cum_bits': 4.3841,\n", " 'delivered_bits': 3.8122,\n", " 'marginal_bits': '[3.0458, 0.7406, 0.4888, 0.1088]',\n", " 'redundancy': 0.2844,\n", " 'cancellation': 0.828,\n", " 'stage_snr': 0.9886,\n", " 'dev_mean': -0.0653,\n", " 'rank_mean': 3.676,\n", " 'killed': '',\n", " 'wall_s': 1.3,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'gate_m4_K16_spread_s1234_391fa6',\n", " 'op': 'gate',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9396,\n", " 'acc': 0.4016,\n", " 'ce_bits': 2.2541,\n", " 'cum_bits': 4.6068,\n", " 'delivered_bits': 3.7459,\n", " 'marginal_bits': '[3.4253, 0.5177, 0.4491, 0.2148]',\n", " 'redundancy': 0.3217,\n", " 'cancellation': 0.6659,\n", " 'stage_snr': 0.674,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 1.1,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m2_K32_free_s1234_77f47d',\n", " 'op': 'res',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 12928,\n", " 'acc': 0.6042,\n", " 'ce_bits': 1.5098,\n", " 'cum_bits': 4.5974,\n", " 'delivered_bits': 4.4902,\n", " 'marginal_bits': '[3.6831, 0.9143]',\n", " 'redundancy': 0.0312,\n", " 'cancellation': 0.4948,\n", " 'stage_snr': 0.3268,\n", " 'dev_mean': -0.0372,\n", " 'rank_mean': 3.804,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m2_K32_spread_s1234_6badfb',\n", " 'op': 'res',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 12928,\n", " 'acc': 0.6851,\n", " 'ce_bits': 1.2895,\n", " 'cum_bits': 4.6999,\n", " 'delivered_bits': 4.7105,\n", " 'marginal_bits': '[3.8575, 0.8423]',\n", " 'redundancy': 0.0495,\n", " 'cancellation': 0.5434,\n", " 'stage_snr': 0.2104,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'res_m3_K21_free_s1234_1925f3',\n", " 'op': 'res',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 12924,\n", " 'acc': 0.729,\n", " 'ce_bits': 1.1054,\n", " 'cum_bits': 4.9963,\n", " 'delivered_bits': 4.8946,\n", " 'marginal_bits': '[3.8044, 0.7096, 0.4822]',\n", " 'redundancy': 0.0681,\n", " 'cancellation': 0.7374,\n", " 'stage_snr': 0.2802,\n", " 'dev_mean': 0.001,\n", " 'rank_mean': 3.887,\n", " 'killed': '',\n", " 'wall_s': 0.8,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'res_m3_K21_spread_s1234_7fd02e',\n", " 'op': 'res',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 12924,\n", " 'acc': 0.7441,\n", " 'ce_bits': 1.0625,\n", " 'cum_bits': 4.9631,\n", " 'delivered_bits': 4.9375,\n", " 'marginal_bits': '[3.7923, 0.8558, 0.315]',\n", " 'redundancy': 0.0692,\n", " 'cancellation': 0.798,\n", " 'stage_snr': 0.2259,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'res_m4_K16_free_s1234_4c2e4d',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 13312,\n", " 'acc': 0.7729,\n", " 'ce_bits': 0.9309,\n", " 'cum_bits': 5.1778,\n", " 'delivered_bits': 5.0691,\n", " 'marginal_bits': '[3.7297, 0.8697, 0.3736, 0.2048]',\n", " 'redundancy': 0.0672,\n", " 'cancellation': 0.881,\n", " 'stage_snr': 0.3302,\n", " 'dev_mean': 0.0172,\n", " 'rank_mean': 3.899,\n", " 'killed': '',\n", " 'wall_s': 1.1,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'res_m4_K16_spread_s1234_0d417f',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 13312,\n", " 'acc': 0.7844,\n", " 'ce_bits': 0.8952,\n", " 'cum_bits': 5.1532,\n", " 'delivered_bits': 5.1048,\n", " 'marginal_bits': '[3.4718, 0.9779, 0.4586, 0.2448]',\n", " 'redundancy': 0.062,\n", " 'cancellation': 0.8869,\n", " 'stage_snr': 0.3003,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 0.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'prod_m2_K32_free_s1234_c2fd6a',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9728,\n", " 'acc': 0.658,\n", " 'ce_bits': 1.3622,\n", " 'cum_bits': 4.6281,\n", " 'delivered_bits': 4.6378,\n", " 'marginal_bits': '[3.5051, 1.123]',\n", " 'redundancy': 0.0356,\n", " 'cancellation': 0.5503,\n", " 'stage_snr': 0.3544,\n", " 'dev_mean': -0.0385,\n", " 'rank_mean': 3.804,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m2_K32_spread_s1234_2a0e45',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9728,\n", " 'acc': 0.6909,\n", " 'ce_bits': 1.2826,\n", " 'cum_bits': 4.6307,\n", " 'delivered_bits': 4.7174,\n", " 'marginal_bits': '[3.7031, 0.9276]',\n", " 'redundancy': 0.071,\n", " 'cancellation': 0.4549,\n", " 'stage_snr': 0.179,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'prod_m3_K21_free_s1234_c5d5b7',\n", " 'op': 'prod',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9660,\n", " 'acc': 0.7434,\n", " 'ce_bits': 1.1297,\n", " 'cum_bits': 4.9881,\n", " 'delivered_bits': 4.8703,\n", " 'marginal_bits': '[3.5967, 1.0109, 0.3805]',\n", " 'redundancy': 0.072,\n", " 'cancellation': 0.769,\n", " 'stage_snr': 0.2797,\n", " 'dev_mean': -0.0207,\n", " 'rank_mean': 3.828,\n", " 'killed': '',\n", " 'wall_s': 0.8,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'prod_m3_K21_spread_s1234_a0d133',\n", " 'op': 'prod',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9660,\n", " 'acc': 0.7646,\n", " 'ce_bits': 1.0266,\n", " 'cum_bits': 5.0217,\n", " 'delivered_bits': 4.9734,\n", " 'marginal_bits': '[3.6619, 1.0069, 0.3529]',\n", " 'redundancy': 0.0463,\n", " 'cancellation': 0.8227,\n", " 'stage_snr': 0.2289,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'prod_m4_K16_free_s1234_be714a',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9856,\n", " 'acc': 0.8086,\n", " 'ce_bits': 0.8652,\n", " 'cum_bits': 5.2411,\n", " 'delivered_bits': 5.1348,\n", " 'marginal_bits': '[3.6226, 0.9476, 0.4001, 0.2708]',\n", " 'redundancy': 0.0705,\n", " 'cancellation': 0.8982,\n", " 'stage_snr': 0.3222,\n", " 'dev_mean': 0.0279,\n", " 'rank_mean': 3.914,\n", " 'killed': '',\n", " 'wall_s': 1.0,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'prod_m4_K16_spread_s1234_77144d',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9856,\n", " 'acc': 0.8201,\n", " 'ce_bits': 0.7882,\n", " 'cum_bits': 5.2431,\n", " 'delivered_bits': 5.2118,\n", " 'marginal_bits': '[3.4752, 1.1586, 0.4234, 0.1859]',\n", " 'redundancy': 0.0844,\n", " 'cancellation': 0.8587,\n", " 'stage_snr': 0.2492,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 0.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'tree_m2_K3_free_s1234_7152c6',\n", " 'op': 'tree',\n", " 'm': 2,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3962,\n", " 'ce_bits': 2.0773,\n", " 'cum_bits': 3.9921,\n", " 'delivered_bits': 3.9227,\n", " 'marginal_bits': '[3.4395, 0.5526]',\n", " 'redundancy': 0.0195,\n", " 'cancellation': 0.6131,\n", " 'stage_snr': 0.5177,\n", " 'dev_mean': 0.1025,\n", " 'rank_mean': 2.785,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m2_K3_hard_free_s1234_42aca9',\n", " 'op': 'tree',\n", " 'm': 2,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': True,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3352,\n", " 'ce_bits': 2.2747,\n", " 'cum_bits': 3.84,\n", " 'delivered_bits': 3.7253,\n", " 'marginal_bits': '[3.2426, 0.5975]',\n", " 'redundancy': 0.0102,\n", " 'cancellation': 0.5833,\n", " 'stage_snr': 0.6481,\n", " 'dev_mean': 0.0719,\n", " 'rank_mean': 2.752,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m2_K3_spread_s1234_667729',\n", " 'op': 'tree',\n", " 'm': 2,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.343,\n", " 'ce_bits': 2.0984,\n", " 'cum_bits': 3.9551,\n", " 'delivered_bits': 3.9016,\n", " 'marginal_bits': '[3.3019, 0.6532]',\n", " 'redundancy': 0.1503,\n", " 'cancellation': 0.5208,\n", " 'stage_snr': 0.4808,\n", " 'dev_mean': 0.4421,\n", " 'rank_mean': 3.057,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m2_K3_hard_spread_s1234_8dff78',\n", " 'op': 'tree',\n", " 'm': 2,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': True,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3159,\n", " 'ce_bits': 2.2203,\n", " 'cum_bits': 3.8386,\n", " 'delivered_bits': 3.7797,\n", " 'marginal_bits': '[3.2043, 0.6343]',\n", " 'redundancy': 0.1504,\n", " 'cancellation': 0.523,\n", " 'stage_snr': 0.4988,\n", " 'dev_mean': 0.4421,\n", " 'rank_mean': 3.057,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m3_K3_free_s1234_da1e88',\n", " 'op': 'tree',\n", " 'm': 3,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3962,\n", " 'ce_bits': 2.0773,\n", " 'cum_bits': 3.9921,\n", " 'delivered_bits': 3.9227,\n", " 'marginal_bits': '[3.4395, 0.5526]',\n", " 'redundancy': 0.0195,\n", " 'cancellation': 0.6131,\n", " 'stage_snr': 0.5177,\n", " 'dev_mean': 0.1025,\n", " 'rank_mean': 2.785,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m3_K3_hard_free_s1234_709495',\n", " 'op': 'tree',\n", " 'm': 3,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': True,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3352,\n", " 'ce_bits': 2.2747,\n", " 'cum_bits': 3.84,\n", " 'delivered_bits': 3.7253,\n", " 'marginal_bits': '[3.2426, 0.5975]',\n", " 'redundancy': 0.0102,\n", " 'cancellation': 0.5833,\n", " 'stage_snr': 0.6481,\n", " 'dev_mean': 0.0719,\n", " 'rank_mean': 2.752,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m3_K3_spread_s1234_574e19',\n", " 'op': 'tree',\n", " 'm': 3,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.343,\n", " 'ce_bits': 2.0984,\n", " 'cum_bits': 3.9551,\n", " 'delivered_bits': 3.9016,\n", " 'marginal_bits': '[3.3019, 0.6532]',\n", " 'redundancy': 0.1503,\n", " 'cancellation': 0.5208,\n", " 'stage_snr': 0.4808,\n", " 'dev_mean': 0.4421,\n", " 'rank_mean': 3.057,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m3_K3_hard_spread_s1234_a37231',\n", " 'op': 'tree',\n", " 'm': 3,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': True,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3159,\n", " 'ce_bits': 2.2203,\n", " 'cum_bits': 3.8386,\n", " 'delivered_bits': 3.7797,\n", " 'marginal_bits': '[3.2043, 0.6343]',\n", " 'redundancy': 0.1504,\n", " 'cancellation': 0.523,\n", " 'stage_snr': 0.4988,\n", " 'dev_mean': 0.4421,\n", " 'rank_mean': 3.057,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m4_K3_free_s1234_5884d1',\n", " 'op': 'tree',\n", " 'm': 4,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3962,\n", " 'ce_bits': 2.0773,\n", " 'cum_bits': 3.9921,\n", " 'delivered_bits': 3.9227,\n", " 'marginal_bits': '[3.4395, 0.5526]',\n", " 'redundancy': 0.0195,\n", " 'cancellation': 0.6131,\n", " 'stage_snr': 0.5177,\n", " 'dev_mean': 0.1025,\n", " 'rank_mean': 2.785,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m4_K3_hard_free_s1234_d48cf2',\n", " 'op': 'tree',\n", " 'm': 4,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': True,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3352,\n", " 'ce_bits': 2.2747,\n", " 'cum_bits': 3.84,\n", " 'delivered_bits': 3.7253,\n", " 'marginal_bits': '[3.2426, 0.5975]',\n", " 'redundancy': 0.0102,\n", " 'cancellation': 0.5833,\n", " 'stage_snr': 0.6481,\n", " 'dev_mean': 0.0719,\n", " 'rank_mean': 2.752,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m4_K3_spread_s1234_5d0ded',\n", " 'op': 'tree',\n", " 'm': 4,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.343,\n", " 'ce_bits': 2.0984,\n", " 'cum_bits': 3.9551,\n", " 'delivered_bits': 3.9016,\n", " 'marginal_bits': '[3.3019, 0.6532]',\n", " 'redundancy': 0.1503,\n", " 'cancellation': 0.5208,\n", " 'stage_snr': 0.4808,\n", " 'dev_mean': 0.4421,\n", " 'rank_mean': 3.057,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'tree_m4_K3_hard_spread_s1234_7f8025',\n", " 'op': 'tree',\n", " 'm': 4,\n", " 'K': 3,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': True,\n", " 'steps': 200,\n", " 'params': 2016,\n", " 'acc': 0.3159,\n", " 'ce_bits': 2.2203,\n", " 'cum_bits': 3.8386,\n", " 'delivered_bits': 3.7797,\n", " 'marginal_bits': '[3.2043, 0.6343]',\n", " 'redundancy': 0.1504,\n", " 'cancellation': 0.523,\n", " 'stage_snr': 0.4988,\n", " 'dev_mean': 0.4421,\n", " 'rank_mean': 3.057,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m2_K32_free_s1234_843ff1',\n", " 'op': 'cross',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 11424,\n", " 'acc': 0.6504,\n", " 'ce_bits': 1.2848,\n", " 'cum_bits': 4.6459,\n", " 'delivered_bits': 4.7152,\n", " 'marginal_bits': '[3.7766, 0.8693]',\n", " 'redundancy': 0.0503,\n", " 'cancellation': 0.5479,\n", " 'stage_snr': 0.3184,\n", " 'dev_mean': -0.0365,\n", " 'rank_mean': 3.818,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'cross_m2_K32_spread_s1234_c82ea9',\n", " 'op': 'cross',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 11424,\n", " 'acc': 0.7017,\n", " 'ce_bits': 1.1513,\n", " 'cum_bits': 4.6501,\n", " 'delivered_bits': 4.8487,\n", " 'marginal_bits': '[3.6905, 0.9596]',\n", " 'redundancy': 0.0423,\n", " 'cancellation': 0.5611,\n", " 'stage_snr': 0.208,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'cross_m3_K21_free_s1234_ae1dcc',\n", " 'op': 'cross',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 12492,\n", " 'acc': 0.7522,\n", " 'ce_bits': 0.987,\n", " 'cum_bits': 4.9497,\n", " 'delivered_bits': 5.013,\n", " 'marginal_bits': '[3.7786, 0.7698, 0.4012]',\n", " 'redundancy': 0.0883,\n", " 'cancellation': 0.7387,\n", " 'stage_snr': 0.3246,\n", " 'dev_mean': -0.0,\n", " 'rank_mean': 3.882,\n", " 'killed': '',\n", " 'wall_s': 0.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'cross_m3_K21_spread_s1234_e95e85',\n", " 'op': 'cross',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 12492,\n", " 'acc': 0.7629,\n", " 'ce_bits': 0.9112,\n", " 'cum_bits': 5.0054,\n", " 'delivered_bits': 5.0888,\n", " 'marginal_bits': '[3.6384, 0.923, 0.444]',\n", " 'redundancy': 0.0504,\n", " 'cancellation': 0.8137,\n", " 'stage_snr': 0.2646,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.8,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'cross_m4_K16_free_s1234_f28e7d',\n", " 'op': 'cross',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 14400,\n", " 'acc': 0.8186,\n", " 'ce_bits': 0.7172,\n", " 'cum_bits': 5.2241,\n", " 'delivered_bits': 5.2828,\n", " 'marginal_bits': '[3.4413, 1.0572, 0.4574, 0.2683]',\n", " 'redundancy': 0.0861,\n", " 'cancellation': 0.8518,\n", " 'stage_snr': 0.4188,\n", " 'dev_mean': 0.0173,\n", " 'rank_mean': 3.884,\n", " 'killed': '',\n", " 'wall_s': 1.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'cross_m4_K16_spread_s1234_d48dff',\n", " 'op': 'cross',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 14400,\n", " 'acc': 0.8101,\n", " 'ce_bits': 0.7686,\n", " 'cum_bits': 5.1767,\n", " 'delivered_bits': 5.2314,\n", " 'marginal_bits': '[3.4292, 1.0595, 0.4405, 0.2475]',\n", " 'redundancy': 0.0792,\n", " 'cancellation': 0.8443,\n", " 'stage_snr': 0.3476,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 1.0,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 16/50 by cum_bits'},\n", " {'arm_id': 'anneal_m2_K64_free_s1234_3cd58b',\n", " 'op': 'anneal',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 16896,\n", " 'acc': 0.4663,\n", " 'ce_bits': 1.8805,\n", " 'cum_bits': 3.9996,\n", " 'delivered_bits': 4.1195,\n", " 'marginal_bits': '[3.6609, 0.3387]',\n", " 'redundancy': 0.9487,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3025,\n", " 'dev_mean': -0.0127,\n", " 'rank_mean': 3.904,\n", " 'killed': '',\n", " 'wall_s': 0.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'anneal_m2_K64_spread_s1234_685a4a',\n", " 'op': 'anneal',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 16896,\n", " 'acc': 0.4824,\n", " 'ce_bits': 1.7909,\n", " 'cum_bits': 3.9822,\n", " 'delivered_bits': 4.2091,\n", " 'marginal_bits': '[3.576, 0.4062]',\n", " 'redundancy': 0.9337,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.1892,\n", " 'dev_mean': 0.0196,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'anneal_m3_K64_free_s1234_27a360',\n", " 'op': 'anneal',\n", " 'm': 3,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 25152,\n", " 'acc': 0.5068,\n", " 'ce_bits': 1.6951,\n", " 'cum_bits': 4.178,\n", " 'delivered_bits': 4.3049,\n", " 'marginal_bits': '[3.6599, 0.4232, 0.0948]',\n", " 'redundancy': 0.9164,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2164,\n", " 'dev_mean': 0.0069,\n", " 'rank_mean': 3.977,\n", " 'killed': '',\n", " 'wall_s': 0.8,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'anneal_m3_K64_spread_s1234_2320dd',\n", " 'op': 'anneal',\n", " 'm': 3,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 25152,\n", " 'acc': 0.5017,\n", " 'ce_bits': 1.7689,\n", " 'cum_bits': 4.089,\n", " 'delivered_bits': 4.2311,\n", " 'marginal_bits': '[3.5418, 0.4811, 0.0661]',\n", " 'redundancy': 0.9075,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.1967,\n", " 'dev_mean': 0.0196,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.7,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'anneal_m4_K64_free_s1234_238ae3',\n", " 'op': 'anneal',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 33408,\n", " 'acc': 0.5061,\n", " 'ce_bits': 1.7808,\n", " 'cum_bits': 4.1442,\n", " 'delivered_bits': 4.2192,\n", " 'marginal_bits': '[3.6047, 0.4819, 0.0304, 0.0272]',\n", " 'redundancy': 0.9013,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.1875,\n", " 'dev_mean': -0.0069,\n", " 'rank_mean': 3.932,\n", " 'killed': '',\n", " 'wall_s': 1.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'anneal_m4_K64_spread_s1234_dd4475',\n", " 'op': 'anneal',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 33408,\n", " 'acc': 0.5161,\n", " 'ce_bits': 1.7434,\n", " 'cum_bits': 4.1296,\n", " 'delivered_bits': 4.2566,\n", " 'marginal_bits': '[3.6392, 0.4584, 0.0201, 0.0118]',\n", " 'redundancy': 0.8979,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.1421,\n", " 'dev_mean': 0.0196,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.9,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8640,\n", " 'acc': 0.4309,\n", " 'ce_bits': 1.9545,\n", " 'cum_bits': 3.8274,\n", " 'delivered_bits': 4.0455,\n", " 'marginal_bits': '[3.8274]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3202,\n", " 'dev_mean': -0.1299,\n", " 'rank_mean': 3.605,\n", " 'killed': '',\n", " 'wall_s': 0.3,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'single_m1_K64_spread_s1234_5dc570',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8640,\n", " 'acc': 0.5034,\n", " 'ce_bits': 1.8552,\n", " 'cum_bits': 3.7269,\n", " 'delivered_bits': 4.1448,\n", " 'marginal_bits': '[3.7269]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2392,\n", " 'dev_mean': 0.0196,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.3,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m4_K16_spread_s1234_77144d',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 9856,\n", " 'acc': 0.8923,\n", " 'ce_bits': 0.4658,\n", " 'cum_bits': 5.362,\n", " 'delivered_bits': 5.5342,\n", " 'marginal_bits': '[3.2781, 1.3449, 0.4821, 0.257]',\n", " 'redundancy': 0.0882,\n", " 'cancellation': 0.8706,\n", " 'stage_snr': 0.2393,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 3.2,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 5/16 by cum_bits'},\n", " {'arm_id': 'cross_m2_K32_free_s1234_843ff1',\n", " 'op': 'cross',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 11424,\n", " 'acc': 0.8018,\n", " 'ce_bits': 0.7928,\n", " 'cum_bits': 4.8287,\n", " 'delivered_bits': 5.2072,\n", " 'marginal_bits': '[3.7992, 1.0295]',\n", " 'redundancy': 0.0676,\n", " 'cancellation': 0.5414,\n", " 'stage_snr': 0.2697,\n", " 'dev_mean': 0.0068,\n", " 'rank_mean': 3.93,\n", " 'killed': '',\n", " 'wall_s': 2.3,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m4_K16_spread_s1234_d48dff',\n", " 'op': 'cross',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 14400,\n", " 'acc': 0.8826,\n", " 'ce_bits': 0.5602,\n", " 'cum_bits': 5.3266,\n", " 'delivered_bits': 5.4398,\n", " 'marginal_bits': '[3.4055, 1.1316, 0.5043, 0.2852]',\n", " 'redundancy': 0.0789,\n", " 'cancellation': 0.8434,\n", " 'stage_snr': 0.265,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 3.7,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m3_K21_spread_s1234_e95e85',\n", " 'op': 'cross',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 12492,\n", " 'acc': 0.8596,\n", " 'ce_bits': 0.5952,\n", " 'cum_bits': 5.18,\n", " 'delivered_bits': 5.4048,\n", " 'marginal_bits': '[3.5683, 1.083, 0.5286]',\n", " 'redundancy': 0.0468,\n", " 'cancellation': 0.8166,\n", " 'stage_snr': 0.2454,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 2.7,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m3_K21_free_s1234_c5d5b7',\n", " 'op': 'prod',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 9660,\n", " 'acc': 0.8582,\n", " 'ce_bits': 0.591,\n", " 'cum_bits': 5.1845,\n", " 'delivered_bits': 5.409,\n", " 'marginal_bits': '[3.3104, 1.2918, 0.5824]',\n", " 'redundancy': 0.0666,\n", " 'cancellation': 0.7585,\n", " 'stage_snr': 0.2715,\n", " 'dev_mean': 0.0289,\n", " 'rank_mean': 3.952,\n", " 'killed': '',\n", " 'wall_s': 3.0,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m3_K21_free_s1234_ae1dcc',\n", " 'op': 'cross',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 12492,\n", " 'acc': 0.8628,\n", " 'ce_bits': 0.6122,\n", " 'cum_bits': 5.1791,\n", " 'delivered_bits': 5.3878,\n", " 'marginal_bits': '[3.7605, 0.9527, 0.466]',\n", " 'redundancy': 0.0882,\n", " 'cancellation': 0.7781,\n", " 'stage_snr': 0.2948,\n", " 'dev_mean': 0.0297,\n", " 'rank_mean': 3.951,\n", " 'killed': '',\n", " 'wall_s': 3.3,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m3_K21_free_s1234_1925f3',\n", " 'op': 'res',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 12924,\n", " 'acc': 0.8577,\n", " 'ce_bits': 0.625,\n", " 'cum_bits': 5.2579,\n", " 'delivered_bits': 5.375,\n", " 'marginal_bits': '[3.839, 0.8904, 0.5285]',\n", " 'redundancy': 0.0817,\n", " 'cancellation': 0.731,\n", " 'stage_snr': 0.223,\n", " 'dev_mean': 0.0249,\n", " 'rank_mean': 3.942,\n", " 'killed': '',\n", " 'wall_s': 3.1,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m4_K16_spread_s1234_0d417f',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 13312,\n", " 'acc': 0.8848,\n", " 'ce_bits': 0.5108,\n", " 'cum_bits': 5.3478,\n", " 'delivered_bits': 5.4892,\n", " 'marginal_bits': '[3.4717, 1.1155, 0.4902, 0.2704]',\n", " 'redundancy': 0.0751,\n", " 'cancellation': 0.8712,\n", " 'stage_snr': 0.2402,\n", " 'dev_mean': 0.0701,\n", " 'rank_mean': 3.996,\n", " 'killed': '',\n", " 'wall_s': 3.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 5/16 by cum_bits'},\n", " {'arm_id': 'prod_m4_K16_free_s1234_be714a',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 9856,\n", " 'acc': 0.895,\n", " 'ce_bits': 0.4705,\n", " 'cum_bits': 5.4027,\n", " 'delivered_bits': 5.5295,\n", " 'marginal_bits': '[3.4495, 1.1775, 0.502, 0.2737]',\n", " 'redundancy': 0.0825,\n", " 'cancellation': 0.9079,\n", " 'stage_snr': 0.3106,\n", " 'dev_mean': 0.036,\n", " 'rank_mean': 3.925,\n", " 'killed': '',\n", " 'wall_s': 4.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 5/16 by cum_bits'},\n", " {'arm_id': 'res_m3_K21_spread_s1234_7fd02e',\n", " 'op': 'res',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 12924,\n", " 'acc': 0.8643,\n", " 'ce_bits': 0.6066,\n", " 'cum_bits': 5.196,\n", " 'delivered_bits': 5.3934,\n", " 'marginal_bits': '[3.6723, 1.0794, 0.4444]',\n", " 'redundancy': 0.0702,\n", " 'cancellation': 0.7912,\n", " 'stage_snr': 0.1955,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 2.6,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m4_K16_free_s1234_f28e7d',\n", " 'op': 'cross',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 14400,\n", " 'acc': 0.8896,\n", " 'ce_bits': 0.5134,\n", " 'cum_bits': 5.3885,\n", " 'delivered_bits': 5.4866,\n", " 'marginal_bits': '[3.5096, 1.0509, 0.4917, 0.3363]',\n", " 'redundancy': 0.0998,\n", " 'cancellation': 0.8548,\n", " 'stage_snr': 0.3507,\n", " 'dev_mean': 0.0413,\n", " 'rank_mean': 3.938,\n", " 'killed': '',\n", " 'wall_s': 4.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 5/16 by cum_bits'},\n", " {'arm_id': 'cross_m2_K32_spread_s1234_c82ea9',\n", " 'op': 'cross',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 11424,\n", " 'acc': 0.8147,\n", " 'ce_bits': 0.7578,\n", " 'cum_bits': 4.8372,\n", " 'delivered_bits': 5.2422,\n", " 'marginal_bits': '[3.6205, 1.2168]',\n", " 'redundancy': 0.0428,\n", " 'cancellation': 0.5731,\n", " 'stage_snr': 0.2028,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 2.2,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m3_K21_spread_s1234_a0d133',\n", " 'op': 'prod',\n", " 'm': 3,\n", " 'K': 21,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 9660,\n", " 'acc': 0.8657,\n", " 'ce_bits': 0.5507,\n", " 'cum_bits': 5.1875,\n", " 'delivered_bits': 5.4493,\n", " 'marginal_bits': '[3.4259, 1.2627, 0.4989]',\n", " 'redundancy': 0.0569,\n", " 'cancellation': 0.8205,\n", " 'stage_snr': 0.1967,\n", " 'dev_mean': 0.0685,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 2.5,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m2_K32_spread_s1234_6badfb',\n", " 'op': 'res',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 12928,\n", " 'acc': 0.8127,\n", " 'ce_bits': 0.7573,\n", " 'cum_bits': 4.8865,\n", " 'delivered_bits': 5.2427,\n", " 'marginal_bits': '[3.9044, 0.9821]',\n", " 'redundancy': 0.061,\n", " 'cancellation': 0.5453,\n", " 'stage_snr': 0.189,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 1.9,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m4_K16_free_s1234_4c2e4d',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 16,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 13312,\n", " 'acc': 0.8906,\n", " 'ce_bits': 0.5034,\n", " 'cum_bits': 5.4072,\n", " 'delivered_bits': 5.4966,\n", " 'marginal_bits': '[3.6992, 1.0163, 0.4287, 0.263]',\n", " 'redundancy': 0.0775,\n", " 'cancellation': 0.8598,\n", " 'stage_snr': 0.2757,\n", " 'dev_mean': 0.029,\n", " 'rank_mean': 3.931,\n", " 'killed': '',\n", " 'wall_s': 4.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 5/16 by cum_bits'},\n", " {'arm_id': 'prod_m2_K32_spread_s1234_2a0e45',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 32,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 9728,\n", " 'acc': 0.8149,\n", " 'ce_bits': 0.7427,\n", " 'cum_bits': 4.8325,\n", " 'delivered_bits': 5.2573,\n", " 'marginal_bits': '[3.4711, 1.3614]',\n", " 'redundancy': 0.0924,\n", " 'cancellation': 0.4542,\n", " 'stage_snr': 0.1659,\n", " 'dev_mean': 0.0266,\n", " 'rank_mean': 3.999,\n", " 'killed': '',\n", " 'wall_s': 1.9,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'}]" ] }, "metadata": {}, "execution_count": 4 } ] }, { "cell_type": "code", "source": [ "fc = ForgeConfig(out_dir=\"exp011b\", push=True, n_train=16384, n_eval=8192,\n", " batch=1024,\n", " bubble=BubbleConfig(d_data=32, branching=(4,4,4,4), sep0=6.0,\n", " sep_decay=0.5, noise=0.3, seed=97))\n", "arms = generate_arms([\"sum\",\"res\",\"prod\",\"cross\"], ms=[12,16],\n", " freezes=[\"free\",\"spread\"], seeds=[1234,5678])\n", "run_screen(arms, fc, rungs=[0,1], protect_ops=(\"sum\",\"single\"))" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "OVsUe_Xzyw7-", "outputId": "13c9b2dc-485c-48bb-f7d6-cb5d61790077" }, "execution_count": 5, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "[rung 0] tier=P steps=200 arms=36 (cached 0, fresh 36)\n", " [1/36] sum_m12_K5_free_s1234_d6fd74 held 4.93 ce 5.168 dlv 2.83 [2.5448, 0.5625, 0.2215, 0.5186, 0.2478, 0.1405, 0.2183, 0.121, 0.0773, 0.0847, 0.1629, 0.0255] acc 0.059 red 0.52\n", " [2/36] sum_m12_K5_free_s5678_f1be97 held 4.67 ce 4.960 dlv 3.04 [2.3553, 0.9687, 0.3695, 0.199, 0.1702, 0.1436, 0.0882, 0.1026, 0.1323, 0.1181, 0.0194, 0.0] acc 0.072 red 0.56\n", " [3/36] sum_m12_K5_spread_s1234_1c84c6 held 4.97 ce 4.718 dlv 3.28 [2.8517, 0.5715, 0.4558, 0.2299, 0.1873, 0.0955, 0.1655, 0.1327, 0.0352, 0.1157, 0.0629, 0.0625] acc 0.074 red 0.42\n", " [4/36] sum_m12_K5_spread_s5678_a6f56c held 4.77 ce 4.779 dlv 3.22 [2.2517, 0.8267, 0.4956, 0.3529, 0.1386, 0.1882, 0.0728, 0.0594, 0.1342, 0.1128, 0.0629, 0.0741] acc 0.088 red 0.46\n", " [5/36] sum_m16_K4_free_s1234_85541a held 4.92 ce 4.692 dlv 3.31 [1.9625, 1.1244, 0.4619, 0.3563, 0.1505, 0.1139, 0.0539, 0.1082, 0.0678, 0.094, 0.0628, 0.089, 0.0222, 0.0443, 0.087, 0.1172] acc 0.078 red 0.56\n", " [6/36] sum_m16_K4_free_s5678_a631de held 4.81 ce 4.799 dlv 3.20 [2.6968, 0.7867, 0.2826, 0.0731, 0.1194, 0.0677, 0.1473, 0.0699, 0.1078, 0.0477, 0.0756, 0.1104, 0.0673, 0.0627, 0.0565, 0.0342] acc 0.073 red 0.68\n", " [7/36] sum_m16_K4_spread_s1234_34c9f7 held 3.98 ce 5.450 dlv 2.55 [2.1692, 0.3836, 0.0998, 0.0, 0.3071, 0.1569, 0.1593, 0.1931, 0.0565, 0.036, 0.105, 0.1325, 0.0156, 0.0634, 0.1017, 0.0] acc 0.033 red 0.58\n", " [8/36] sum_m16_K4_spread_s5678_179916 held 4.52 ce 5.145 dlv 2.86 [2.3256, 0.1496, 0.5335, 0.4369, 0.2889, 0.0155, 0.0617, 0.0196, 0.1015, 0.2969, 0.0621, 0.0013, 0.0798, 0.0393, 0.0246, 0.079] acc 0.042 red 0.54\n", " [9/36] res_m12_K5_free_s1234_2186b5 held 6.27 ce 1.832 dlv 6.17 [2.6884, 1.3857, 0.6034, 0.4452, 0.3668, 0.2456, 0.1675, 0.1203, 0.0517, 0.0963, 0.0639, 0.0304] acc 0.510 red 0.14\n", " [10/36] res_m12_K5_free_s5678_66ab41 held 6.25 ce 1.825 dlv 6.17 [2.2278, 1.7415, 0.7308, 0.5106, 0.3314, 0.1788, 0.1263, 0.1329, 0.0966, 0.0805, 0.0708, 0.0225] acc 0.514 red 0.14\n", " [11/36] res_m12_K5_spread_s1234_83f2f5 held 6.29 ce 1.783 dlv 6.22 [2.699, 1.449, 0.7447, 0.4526, 0.2729, 0.1528, 0.1945, 0.1009, 0.0697, 0.0731, 0.0323, 0.0523] acc 0.521 red 0.13\n", " [12/36] res_m12_K5_spread_s5678_c373bf held 6.24 ce 1.826 dlv 6.17 [2.4802, 1.6425, 0.7315, 0.353, 0.3474, 0.1424, 0.1206, 0.1308, 0.1074, 0.0842, 0.0532, 0.0485] acc 0.500 red 0.14\n", " [13/36] res_m16_K4_free_s1234_52ba39 held 6.38 ce 1.708 dlv 6.29 [2.5138, 1.3203, 0.7899, 0.4756, 0.3276, 0.2039, 0.1672, 0.1026, 0.1187, 0.1084, 0.0631, 0.0485, 0.0016, 0.0763, 0.0178, 0.0477] acc 0.541 red 0.17\n", " [14/36] res_m16_K4_free_s5678_d700ae held 6.37 ce 1.758 dlv 6.24 [2.3895, 1.2923, 0.8345, 0.4482, 0.3299, 0.2018, 0.1728, 0.1223, 0.173, 0.0659, 0.1244, 0.021, 0.0848, 0.0564, 0.0373, 0.0188] acc 0.527 red 0.18\n", " [15/36] res_m16_K4_spread_s1234_190427 held 6.37 ce 1.718 dlv 6.28 [2.4183, 1.3995, 0.6132, 0.435, 0.3148, 0.2678, 0.2368, 0.1607, 0.0818, 0.136, 0.0858, 0.0455, 0.0351, 0.0494, 0.0474, 0.0461] acc 0.533 red 0.16\n", " [16/36] res_m16_K4_spread_s5678_761442 held 6.38 ce 1.726 dlv 6.27 [2.2839, 1.3639, 0.7126, 0.5376, 0.3546, 0.2387, 0.2125, 0.1342, 0.0998, 0.123, 0.05, 0.075, 0.0783, 0.0532, 0.0257, 0.039] acc 0.529 red 0.16\n", " [17/36] prod_m12_K5_free_s1234_2dcb85 held 6.20 ce 1.775 dlv 6.23 [1.2668, 1.9047, 1.2154, 0.383, 0.5487, 0.25, 0.0741, 0.2252, 0.1344, 0.0746, 0.088, 0.0318] acc 0.528 red 0.14\n", " [18/36] prod_m12_K5_free_s5678_231c6e held 6.20 ce 1.751 dlv 6.25 [1.5988, 1.8636, 1.0148, 0.3814, 0.4514, 0.242, 0.1051, 0.1516, 0.175, 0.0722, 0.0933, 0.0547] acc 0.535 red 0.15\n", " [19/36] prod_m12_K5_spread_s1234_0ed10a held 6.19 ce 1.802 dlv 6.20 [1.353, 1.8523, 1.2126, 0.4158, 0.4231, 0.2539, 0.153, 0.2171, 0.0407, 0.0665, 0.1102, 0.091] acc 0.533 red 0.13\n", " [20/36] prod_m12_K5_spread_s5678_524239 held 6.22 ce 1.838 dlv 6.16 [1.7956, 1.7622, 0.8782, 0.3191, 0.4495, 0.3122, 0.148, 0.1815, 0.1541, 0.0412, 0.1242, 0.0497] acc 0.513 red 0.14\n", " [21/36] prod_m16_K4_free_s1234_16391d held 6.22 ce 1.741 dlv 6.26 [1.8978, 1.2734, 0.5585, 0.6955, 0.3719, 0.3312, 0.1723, 0.2173, 0.1418, 0.1479, 0.0959, 0.1296, 0.0272, 0.0713, 0.0371, 0.0557] acc 0.557 red 0.17\n", " [22/36] prod_m16_K4_free_s5678_3f9996 held 6.19 ce 1.709 dlv 6.29 [1.8559, 1.4146, 0.5753, 0.6134, 0.4027, 0.1934, 0.2009, 0.1432, 0.1614, 0.1049, 0.1561, 0.0897, 0.1201, 0.0257, 0.1289, 0.0072] acc 0.561 red 0.16\n", " [23/36] prod_m16_K4_spread_s1234_ddb49b held 6.24 ce 1.724 dlv 6.28 [1.8829, 1.3348, 0.8193, 0.4072, 0.4622, 0.2444, 0.2636, 0.1855, 0.1243, 0.122, 0.0985, 0.0752, 0.0255, 0.0922, 0.0446, 0.0594] acc 0.555 red 0.17\n", " [24/36] prod_m16_K4_spread_s5678_260ac1 held 6.20 ce 1.760 dlv 6.24 [1.4394, 1.1852, 0.7993, 0.7731, 0.5132, 0.3514, 0.2487, 0.1951, 0.1132, 0.1399, 0.1576, 0.0786, 0.098, 0.0228, 0.0833, 0.0] acc 0.543 red 0.16\n", " [25/36] cross_m12_K5_free_s1234_41ba1d held 6.40 ce 1.436 dlv 6.56 [2.5624, 1.2671, 0.7282, 0.4558, 0.4339, 0.2603, 0.1814, 0.1411, 0.0931, 0.1291, 0.0587, 0.0859] acc 0.619 red 0.15\n", " [26/36] cross_m12_K5_free_s5678_d62bfc held 6.39 ce 1.436 dlv 6.56 [2.1549, 1.7407, 0.954, 0.4098, 0.2935, 0.2181, 0.1499, 0.1411, 0.1146, 0.114, 0.0733, 0.0251] acc 0.621 red 0.16\n", " [27/36] cross_m12_K5_spread_s1234_346e68 held 6.41 ce 1.446 dlv 6.55 [2.5694, 1.3935, 0.8399, 0.3745, 0.3321, 0.2357, 0.1408, 0.1377, 0.1171, 0.104, 0.0638, 0.0973] acc 0.618 red 0.14\n", " [28/36] cross_m12_K5_spread_s5678_d04d82 held 6.35 ce 1.470 dlv 6.53 [2.2157, 1.6542, 0.8482, 0.41, 0.3625, 0.21, 0.1498, 0.1647, 0.0719, 0.1023, 0.0844, 0.0716] acc 0.613 red 0.16\n", " [29/36] cross_m16_K4_free_s1234_c67f01 held 6.48 ce 1.359 dlv 6.64 [2.0315, 1.4738, 0.9752, 0.4626, 0.446, 0.2459, 0.18, 0.1154, 0.1039, 0.0992, 0.0979, 0.0775, 0.01, 0.0728, 0.067, 0.0176] acc 0.651 red 0.19\n", " [30/36] cross_m16_K4_free_s5678_edce59 held 6.47 ce 1.311 dlv 6.69 [1.9963, 1.4521, 0.9066, 0.5418, 0.3509, 0.2195, 0.2372, 0.1394, 0.1147, 0.1269, 0.1519, 0.0184, 0.088, 0.056, 0.0571, 0.0176] acc 0.660 red 0.19\n", " [31/36] cross_m16_K4_spread_s1234_b01a9b held 6.46 ce 1.353 dlv 6.65 [2.1028, 1.3648, 0.8906, 0.5949, 0.2653, 0.3467, 0.18, 0.1605, 0.1285, 0.1223, 0.081, 0.0577, 0.0462, 0.0339, 0.0523, 0.0375] acc 0.652 red 0.17\n", " [32/36] cross_m16_K4_spread_s5678_ee53da held 6.53 ce 1.319 dlv 6.68 [2.0934, 1.3997, 0.8913, 0.4767, 0.3722, 0.2896, 0.2358, 0.1426, 0.1574, 0.0938, 0.1345, 0.0263, 0.0541, 0.0723, 0.0593, 0.0304] acc 0.663 red 0.19\n", " [33/36] single_m1_K64_free_s1234_194318 held 3.47 ce 3.852 dlv 4.15 [3.4673] acc 0.159 red 0.00\n", " [34/36] single_m1_K64_free_s5678_800880 held 3.19 ce 3.824 dlv 4.18 [3.1941] acc 0.164 red 0.00\n", " [35/36] single_m1_K64_spread_s1234_5dc570 held 2.85 ce 3.484 dlv 4.52 [2.8509] acc 0.221 red 0.00\n", " [36/36] single_m1_K64_spread_s5678_338545 held 2.77 ce 3.549 dlv 4.45 [2.7722] acc 0.222 red 0.00\n", "[push] exp011b/ -> AbstractPhil/geolip-aleph-differentiation ✓\n", "\n", "[rung 1] tier=P steps=1000 arms=24 (cached 0, fresh 24)\n", " [1/24] cross_m12_K5_free_s1234_41ba1d held 6.57 ce 1.409 dlv 6.59 [2.4615, 1.32, 0.7649, 0.5384, 0.4133, 0.2699, 0.218, 0.1618, 0.1331, 0.1094, 0.0905, 0.091] acc 0.719 red 0.17\n", " [2/24] cross_m16_K4_free_s1234_c67f01 held 6.64 ce 1.560 dlv 6.44 [1.8261, 1.5147, 1.105, 0.5414, 0.4241, 0.2561, 0.2533, 0.1259, 0.1145, 0.0993, 0.0781, 0.0935, 0.0428, 0.0754, 0.0606, 0.0274] acc 0.726 red 0.21\n", " [3/24] single_m1_K64_free_s1234_194318 held 3.54 ce 2.692 dlv 5.31 [3.5431] acc 0.305 red 0.00\n", " [4/24] single_m1_K64_spread_s1234_5dc570 held 2.95 ce 2.645 dlv 5.35 [2.9511] acc 0.315 red 0.00\n", " [5/24] sum_m12_K5_spread_s1234_1c84c6 held 6.29 ce 1.945 dlv 6.06 [2.6624, 1.3563, 0.8241, 0.4233, 0.3055, 0.1615, 0.1862, 0.1151, 0.0199, 0.121, 0.0471, 0.0724] acc 0.470 red 0.32\n", " [6/24] sum_m16_K4_spread_s1234_34c9f7 held 6.25 ce 2.149 dlv 5.85 [2.223, 1.3927, 0.5685, 0.486, 0.4527, 0.1659, 0.2798, 0.1631, 0.0992, 0.1025, 0.0998, 0.0539, 0.0262, 0.099, 0.0, 0.0395] acc 0.414 red 0.39\n", " [7/24] res_m16_K4_spread_s1234_190427 held 6.63 ce 1.277 dlv 6.72 [2.4488, 1.2806, 0.7416, 0.5198, 0.3425, 0.2847, 0.2445, 0.1801, 0.1089, 0.1282, 0.0612, 0.0819, 0.0218, 0.091, 0.0306, 0.064] acc 0.696 red 0.18\n", " [8/24] sum_m16_K4_free_s5678_a631de held 6.30 ce 2.087 dlv 5.91 [2.4988, 1.3273, 0.7245, 0.4308, 0.3731, 0.1173, 0.1541, 0.1329, 0.1295, 0.1058, 0.052, 0.0604, 0.0814, 0.0364, 0.0738, 0.0] acc 0.423 red 0.42\n", " [9/24] cross_m16_K4_spread_s1234_b01a9b held 6.66 ce 1.599 dlv 6.40 [2.0986, 1.4003, 0.9186, 0.6137, 0.3039, 0.3379, 0.1957, 0.1544, 0.1484, 0.1231, 0.0715, 0.0768, 0.0503, 0.0399, 0.0533, 0.0706] acc 0.714 red 0.19\n", " [10/24] res_m16_K4_spread_s5678_761442 held 6.62 ce 1.342 dlv 6.66 [2.3219, 1.3294, 0.8387, 0.551, 0.3483, 0.2429, 0.2211, 0.1367, 0.1006, 0.136, 0.1013, 0.0435, 0.077, 0.0853, 0.0514, 0.0332] acc 0.700 red 0.18\n", " [11/24] res_m16_K4_free_s5678_d700ae held 6.64 ce 1.267 dlv 6.73 [2.3748, 1.3549, 0.8591, 0.4468, 0.3895, 0.2311, 0.2017, 0.1274, 0.1685, 0.1055, 0.1192, 0.0015, 0.1274, 0.0474, 0.0695, 0.0176] acc 0.702 red 0.19\n", " [12/24] single_m1_K64_spread_s5678_338545 held 2.90 ce 2.662 dlv 5.34 [2.9003] acc 0.316 red 0.00\n", " [13/24] sum_m12_K5_free_s1234_d6fd74 held 6.28 ce 1.961 dlv 6.04 [2.421, 1.5628, 0.618, 0.4326, 0.4234, 0.3161, 0.1535, 0.1277, 0.0413, 0.0777, 0.0653, 0.0444] acc 0.464 red 0.37\n", " [14/24] cross_m12_K5_spread_s1234_346e68 held 6.56 ce 1.451 dlv 6.55 [2.5788, 1.4416, 0.814, 0.4013, 0.3668, 0.2502, 0.169, 0.1109, 0.1101, 0.1362, 0.0816, 0.0994] acc 0.703 red 0.16\n", " [15/24] single_m1_K64_free_s5678_800880 held 3.17 ce 2.644 dlv 5.36 [3.1654] acc 0.326 red 0.00\n", " [16/24] res_m16_K4_free_s1234_52ba39 held 6.62 ce 1.296 dlv 6.70 [2.484, 1.2902, 0.8344, 0.5119, 0.3798, 0.2305, 0.194, 0.1542, 0.0944, 0.1267, 0.097, 0.0303, 0.055, 0.0627, 0.0074, 0.0674] acc 0.701 red 0.18\n", " [17/24] sum_m16_K4_free_s1234_85541a held 6.38 ce 1.960 dlv 6.04 [2.0197, 1.5604, 0.9234, 0.549, 0.3496, 0.2922, 0.1365, 0.1184, 0.0822, 0.1053, 0.0582, 0.0479, 0.0314, 0.0397, 0.0348, 0.0361] acc 0.478 red 0.38\n", " [18/24] sum_m12_K5_free_s5678_f1be97 held 6.27 ce 1.982 dlv 6.02 [2.3669, 1.7378, 0.6613, 0.4194, 0.2719, 0.197, 0.1508, 0.1933, 0.0988, 0.0767, 0.0792, 0.0202] acc 0.479 red 0.36\n", " [19/24] cross_m16_K4_spread_s5678_ee53da held 6.67 ce 1.558 dlv 6.44 [2.0578, 1.3765, 1.0057, 0.5062, 0.3913, 0.2875, 0.2272, 0.1739, 0.1436, 0.0924, 0.1493, 0.0513, 0.0645, 0.0641, 0.0682, 0.0126] acc 0.723 red 0.20\n", " [20/24] cross_m12_K5_spread_s5678_d04d82 held 6.52 ce 1.473 dlv 6.53 [2.2159, 1.607, 0.8728, 0.4463, 0.3923, 0.2763, 0.1616, 0.1745, 0.103, 0.121, 0.0938, 0.0575] acc 0.712 red 0.18\n", " [21/24] sum_m12_K5_spread_s5678_a6f56c held 6.23 ce 2.017 dlv 5.98 [2.4996, 1.5549, 0.7027, 0.4346, 0.2568, 0.1145, 0.1756, 0.1241, 0.1397, 0.0995, 0.0719, 0.0542] acc 0.464 red 0.34\n", " [22/24] sum_m16_K4_spread_s5678_179916 held 6.33 ce 2.038 dlv 5.96 [2.5259, 1.2685, 0.7107, 0.626, 0.2913, 0.165, 0.1977, 0.094, 0.0907, 0.0864, 0.0803, 0.0305, 0.0606, 0.0053, 0.0798, 0.0135] acc 0.447 red 0.36\n", " [23/24] cross_m12_K5_free_s5678_d62bfc held 6.52 ce 1.429 dlv 6.57 [2.123, 1.7879, 0.9346, 0.4333, 0.3286, 0.2262, 0.1872, 0.1432, 0.1253, 0.117, 0.0915, 0.0244] acc 0.715 red 0.17\n", " [24/24] cross_m16_K4_free_s5678_edce59 held 6.61 ce 1.614 dlv 6.39 [1.9679, 1.373, 1.0335, 0.5241, 0.4089, 0.2683, 0.2176, 0.1843, 0.1192, 0.1241, 0.1305, 0.025, 0.1026, 0.0472, 0.0696, 0.0097] acc 0.725 red 0.21\n", "[push] exp011b/ -> AbstractPhil/geolip-aleph-differentiation ✓\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "[{'arm_id': 'sum_m12_K5_free_s1234_d6fd74',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10236,\n", " 'acc': 0.0586,\n", " 'ce_bits': 5.1677,\n", " 'cum_bits': 4.9255,\n", " 'delivered_bits': 2.8323,\n", " 'marginal_bits': '[2.5448, 0.5625, 0.2215, 0.5186, 0.2478, 0.1405, 0.2183, 0.121, 0.0773, 0.0847, 0.1629, 0.0255]',\n", " 'redundancy': 0.5166,\n", " 'cancellation': 0.2092,\n", " 'stage_snr': 1.0952,\n", " 'dev_mean': 0.1891,\n", " 'rank_mean': 3.853,\n", " 'killed': '',\n", " 'wall_s': 4.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m12_K5_free_s5678_f1be97',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10236,\n", " 'acc': 0.0721,\n", " 'ce_bits': 4.9598,\n", " 'cum_bits': 4.6669,\n", " 'delivered_bits': 3.0402,\n", " 'marginal_bits': '[2.3553, 0.9687, 0.3695, 0.199, 0.1702, 0.1436, 0.0882, 0.1026, 0.1323, 0.1181, 0.0194, 0.0]',\n", " 'redundancy': 0.5565,\n", " 'cancellation': 0.2501,\n", " 'stage_snr': 1.3814,\n", " 'dev_mean': 0.1533,\n", " 'rank_mean': 3.779,\n", " 'killed': '',\n", " 'wall_s': 4.6,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m12_K5_spread_s1234_1c84c6',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10236,\n", " 'acc': 0.0737,\n", " 'ce_bits': 4.718,\n", " 'cum_bits': 4.9662,\n", " 'delivered_bits': 3.282,\n", " 'marginal_bits': '[2.8517, 0.5715, 0.4558, 0.2299, 0.1873, 0.0955, 0.1655, 0.1327, 0.0352, 0.1157, 0.0629, 0.0625]',\n", " 'redundancy': 0.4241,\n", " 'cancellation': 0.272,\n", " 'stage_snr': 1.0444,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 4.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m12_K5_spread_s5678_a6f56c',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10236,\n", " 'acc': 0.0879,\n", " 'ce_bits': 4.7792,\n", " 'cum_bits': 4.77,\n", " 'delivered_bits': 3.2208,\n", " 'marginal_bits': '[2.2517, 0.8267, 0.4956, 0.3529, 0.1386, 0.1882, 0.0728, 0.0594, 0.1342, 0.1128, 0.0629, 0.0741]',\n", " 'redundancy': 0.4627,\n", " 'cancellation': 0.5087,\n", " 'stage_snr': 1.2262,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 4.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m16_K4_free_s1234_85541a',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 11536,\n", " 'acc': 0.0779,\n", " 'ce_bits': 4.6919,\n", " 'cum_bits': 4.9159,\n", " 'delivered_bits': 3.3081,\n", " 'marginal_bits': '[1.9625, 1.1244, 0.4619, 0.3563, 0.1505, 0.1139, 0.0539, 0.1082, 0.0678, 0.094, 0.0628, 0.089, 0.0222, 0.0443, 0.087, 0.1172]',\n", " 'redundancy': 0.5596,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 1.0281,\n", " 'dev_mean': 0.2071,\n", " 'rank_mean': 3.657,\n", " 'killed': '',\n", " 'wall_s': 6.3,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m16_K4_free_s5678_a631de',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 11536,\n", " 'acc': 0.073,\n", " 'ce_bits': 4.7992,\n", " 'cum_bits': 4.8056,\n", " 'delivered_bits': 3.2008,\n", " 'marginal_bits': '[2.6968, 0.7867, 0.2826, 0.0731, 0.1194, 0.0677, 0.1473, 0.0699, 0.1078, 0.0477, 0.0756, 0.1104, 0.0673, 0.0627, 0.0565, 0.0342]',\n", " 'redundancy': 0.6848,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 1.2029,\n", " 'dev_mean': 0.202,\n", " 'rank_mean': 3.659,\n", " 'killed': '',\n", " 'wall_s': 6.3,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m16_K4_spread_s1234_34c9f7',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 11536,\n", " 'acc': 0.0327,\n", " 'ce_bits': 5.4497,\n", " 'cum_bits': 3.9798,\n", " 'delivered_bits': 2.5503,\n", " 'marginal_bits': '[2.1692, 0.3836, 0.0998, 0.0, 0.3071, 0.1569, 0.1593, 0.1931, 0.0565, 0.036, 0.105, 0.1325, 0.0156, 0.0634, 0.1017, 0.0]',\n", " 'redundancy': 0.579,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 2.6564,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 5.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m16_K4_spread_s5678_179916',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 11536,\n", " 'acc': 0.0415,\n", " 'ce_bits': 5.1446,\n", " 'cum_bits': 4.5158,\n", " 'delivered_bits': 2.8554,\n", " 'marginal_bits': '[2.3256, 0.1496, 0.5335, 0.4369, 0.2889, 0.0155, 0.0617, 0.0196, 0.1015, 0.2969, 0.0621, 0.0013, 0.0798, 0.0393, 0.0246, 0.079]',\n", " 'redundancy': 0.5386,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 1.9957,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 5.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m12_K5_free_s1234_2186b5',\n", " 'op': 'res',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 14064,\n", " 'acc': 0.5104,\n", " 'ce_bits': 1.8323,\n", " 'cum_bits': 6.2652,\n", " 'delivered_bits': 6.1677,\n", " 'marginal_bits': '[2.6884, 1.3857, 0.6034, 0.4452, 0.3668, 0.2456, 0.1675, 0.1203, 0.0517, 0.0963, 0.0639, 0.0304]',\n", " 'redundancy': 0.1368,\n", " 'cancellation': 0.3743,\n", " 'stage_snr': 0.4033,\n", " 'dev_mean': 0.1523,\n", " 'rank_mean': 3.759,\n", " 'killed': '',\n", " 'wall_s': 4.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m12_K5_free_s5678_66ab41',\n", " 'op': 'res',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 14064,\n", " 'acc': 0.5143,\n", " 'ce_bits': 1.825,\n", " 'cum_bits': 6.2503,\n", " 'delivered_bits': 6.175,\n", " 'marginal_bits': '[2.2278, 1.7415, 0.7308, 0.5106, 0.3314, 0.1788, 0.1263, 0.1329, 0.0966, 0.0805, 0.0708, 0.0225]',\n", " 'redundancy': 0.1437,\n", " 'cancellation': 0.3625,\n", " 'stage_snr': 0.3387,\n", " 'dev_mean': 0.1242,\n", " 'rank_mean': 3.681,\n", " 'killed': '',\n", " 'wall_s': 4.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m12_K5_spread_s1234_83f2f5',\n", " 'op': 'res',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 14064,\n", " 'acc': 0.521,\n", " 'ce_bits': 1.7834,\n", " 'cum_bits': 6.2938,\n", " 'delivered_bits': 6.2166,\n", " 'marginal_bits': '[2.699, 1.449, 0.7447, 0.4526, 0.2729, 0.1528, 0.1945, 0.1009, 0.0697, 0.0731, 0.0323, 0.0523]',\n", " 'redundancy': 0.1271,\n", " 'cancellation': 0.4008,\n", " 'stage_snr': 0.374,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 4.1,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m12_K5_spread_s5678_c373bf',\n", " 'op': 'res',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 14064,\n", " 'acc': 0.5001,\n", " 'ce_bits': 1.8258,\n", " 'cum_bits': 6.2417,\n", " 'delivered_bits': 6.1742,\n", " 'marginal_bits': '[2.4802, 1.6425, 0.7315, 0.353, 0.3474, 0.1424, 0.1206, 0.1308, 0.1074, 0.0842, 0.0532, 0.0485]',\n", " 'redundancy': 0.1363,\n", " 'cancellation': 0.4215,\n", " 'stage_snr': 0.353,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 4.1,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m16_K4_free_s1234_52ba39',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 15616,\n", " 'acc': 0.541,\n", " 'ce_bits': 1.7081,\n", " 'cum_bits': 6.3831,\n", " 'delivered_bits': 6.2919,\n", " 'marginal_bits': '[2.5138, 1.3203, 0.7899, 0.4756, 0.3276, 0.2039, 0.1672, 0.1026, 0.1187, 0.1084, 0.0631, 0.0485, 0.0016, 0.0763, 0.0178, 0.0477]',\n", " 'redundancy': 0.17,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3343,\n", " 'dev_mean': 0.1427,\n", " 'rank_mean': 3.485,\n", " 'killed': '',\n", " 'wall_s': 6.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_free_s5678_d700ae',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 15616,\n", " 'acc': 0.5271,\n", " 'ce_bits': 1.7583,\n", " 'cum_bits': 6.373,\n", " 'delivered_bits': 6.2417,\n", " 'marginal_bits': '[2.3895, 1.2923, 0.8345, 0.4482, 0.3299, 0.2018, 0.1728, 0.1223, 0.173, 0.0659, 0.1244, 0.021, 0.0848, 0.0564, 0.0373, 0.0188]',\n", " 'redundancy': 0.1784,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3599,\n", " 'dev_mean': 0.1583,\n", " 'rank_mean': 3.539,\n", " 'killed': '',\n", " 'wall_s': 7.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_spread_s1234_190427',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 15616,\n", " 'acc': 0.5332,\n", " 'ce_bits': 1.7182,\n", " 'cum_bits': 6.3732,\n", " 'delivered_bits': 6.2818,\n", " 'marginal_bits': '[2.4183, 1.3995, 0.6132, 0.435, 0.3148, 0.2678, 0.2368, 0.1607, 0.0818, 0.136, 0.0858, 0.0455, 0.0351, 0.0494, 0.0474, 0.0461]',\n", " 'redundancy': 0.1629,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4309,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 6.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_spread_s5678_761442',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 15616,\n", " 'acc': 0.5294,\n", " 'ce_bits': 1.7259,\n", " 'cum_bits': 6.382,\n", " 'delivered_bits': 6.2741,\n", " 'marginal_bits': '[2.2839, 1.3639, 0.7126, 0.5376, 0.3546, 0.2387, 0.2125, 0.1342, 0.0998, 0.123, 0.05, 0.075, 0.0783, 0.0532, 0.0257, 0.039]',\n", " 'redundancy': 0.1616,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4173,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 6.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'prod_m12_K5_free_s1234_2dcb85',\n", " 'op': 'prod',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9840,\n", " 'acc': 0.5282,\n", " 'ce_bits': 1.7748,\n", " 'cum_bits': 6.1966,\n", " 'delivered_bits': 6.2252,\n", " 'marginal_bits': '[1.2668, 1.9047, 1.2154, 0.383, 0.5487, 0.25, 0.0741, 0.2252, 0.1344, 0.0746, 0.088, 0.0318]',\n", " 'redundancy': 0.1387,\n", " 'cancellation': 0.384,\n", " 'stage_snr': 0.3979,\n", " 'dev_mean': 0.1542,\n", " 'rank_mean': 3.728,\n", " 'killed': '',\n", " 'wall_s': 4.4,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m12_K5_free_s5678_231c6e',\n", " 'op': 'prod',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9840,\n", " 'acc': 0.5354,\n", " 'ce_bits': 1.7507,\n", " 'cum_bits': 6.204,\n", " 'delivered_bits': 6.2493,\n", " 'marginal_bits': '[1.5988, 1.8636, 1.0148, 0.3814, 0.4514, 0.242, 0.1051, 0.1516, 0.175, 0.0722, 0.0933, 0.0547]',\n", " 'redundancy': 0.1466,\n", " 'cancellation': 0.4268,\n", " 'stage_snr': 0.3972,\n", " 'dev_mean': 0.0838,\n", " 'rank_mean': 3.52,\n", " 'killed': '',\n", " 'wall_s': 4.4,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m12_K5_spread_s1234_0ed10a',\n", " 'op': 'prod',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9840,\n", " 'acc': 0.5333,\n", " 'ce_bits': 1.8018,\n", " 'cum_bits': 6.1892,\n", " 'delivered_bits': 6.1982,\n", " 'marginal_bits': '[1.353, 1.8523, 1.2126, 0.4158, 0.4231, 0.2539, 0.153, 0.2171, 0.0407, 0.0665, 0.1102, 0.091]',\n", " 'redundancy': 0.1308,\n", " 'cancellation': 0.3916,\n", " 'stage_snr': 0.42,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 4.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m12_K5_spread_s5678_524239',\n", " 'op': 'prod',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 9840,\n", " 'acc': 0.5132,\n", " 'ce_bits': 1.838,\n", " 'cum_bits': 6.2153,\n", " 'delivered_bits': 6.162,\n", " 'marginal_bits': '[1.7956, 1.7622, 0.8782, 0.3191, 0.4495, 0.3122, 0.148, 0.1815, 0.1541, 0.0412, 0.1242, 0.0497]',\n", " 'redundancy': 0.141,\n", " 'cancellation': 0.4561,\n", " 'stage_snr': 0.4405,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 4.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m16_K4_free_s1234_16391d',\n", " 'op': 'prod',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10624,\n", " 'acc': 0.5568,\n", " 'ce_bits': 1.7409,\n", " 'cum_bits': 6.2244,\n", " 'delivered_bits': 6.2591,\n", " 'marginal_bits': '[1.8978, 1.2734, 0.5585, 0.6955, 0.3719, 0.3312, 0.1723, 0.2173, 0.1418, 0.1479, 0.0959, 0.1296, 0.0272, 0.0713, 0.0371, 0.0557]',\n", " 'redundancy': 0.1691,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.445,\n", " 'dev_mean': 0.1245,\n", " 'rank_mean': 3.221,\n", " 'killed': '',\n", " 'wall_s': 6.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m16_K4_free_s5678_3f9996',\n", " 'op': 'prod',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10624,\n", " 'acc': 0.5614,\n", " 'ce_bits': 1.7091,\n", " 'cum_bits': 6.1936,\n", " 'delivered_bits': 6.2909,\n", " 'marginal_bits': '[1.8559, 1.4146, 0.5753, 0.6134, 0.4027, 0.1934, 0.2009, 0.1432, 0.1614, 0.1049, 0.1561, 0.0897, 0.1201, 0.0257, 0.1289, 0.0072]',\n", " 'redundancy': 0.1646,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3891,\n", " 'dev_mean': 0.1335,\n", " 'rank_mean': 3.349,\n", " 'killed': '',\n", " 'wall_s': 6.0,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m16_K4_spread_s1234_ddb49b',\n", " 'op': 'prod',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10624,\n", " 'acc': 0.5548,\n", " 'ce_bits': 1.7241,\n", " 'cum_bits': 6.2417,\n", " 'delivered_bits': 6.2759,\n", " 'marginal_bits': '[1.8829, 1.3348, 0.8193, 0.4072, 0.4622, 0.2444, 0.2636, 0.1855, 0.1243, 0.122, 0.0985, 0.0752, 0.0255, 0.0922, 0.0446, 0.0594]',\n", " 'redundancy': 0.1687,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4226,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 5.4,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m16_K4_spread_s5678_260ac1',\n", " 'op': 'prod',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 10624,\n", " 'acc': 0.5427,\n", " 'ce_bits': 1.7603,\n", " 'cum_bits': 6.1987,\n", " 'delivered_bits': 6.2397,\n", " 'marginal_bits': '[1.4394, 1.1852, 0.7993, 0.7731, 0.5132, 0.3514, 0.2487, 0.1951, 0.1132, 0.1399, 0.1576, 0.0786, 0.098, 0.0228, 0.0833, 0.0]',\n", " 'redundancy': 0.1604,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3765,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 5.5,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m12_K5_free_s1234_41ba1d',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 46128,\n", " 'acc': 0.6194,\n", " 'ce_bits': 1.4359,\n", " 'cum_bits': 6.3969,\n", " 'delivered_bits': 6.5641,\n", " 'marginal_bits': '[2.5624, 1.2671, 0.7282, 0.4558, 0.4339, 0.2603, 0.1814, 0.1411, 0.0931, 0.1291, 0.0587, 0.0859]',\n", " 'redundancy': 0.153,\n", " 'cancellation': 0.3655,\n", " 'stage_snr': 0.5704,\n", " 'dev_mean': 0.1339,\n", " 'rank_mean': 3.669,\n", " 'killed': '',\n", " 'wall_s': 5.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m12_K5_free_s5678_d62bfc',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 46128,\n", " 'acc': 0.621,\n", " 'ce_bits': 1.4362,\n", " 'cum_bits': 6.389,\n", " 'delivered_bits': 6.5638,\n", " 'marginal_bits': '[2.1549, 1.7407, 0.954, 0.4098, 0.2935, 0.2181, 0.1499, 0.1411, 0.1146, 0.114, 0.0733, 0.0251]',\n", " 'redundancy': 0.1564,\n", " 'cancellation': 0.3626,\n", " 'stage_snr': 0.4991,\n", " 'dev_mean': 0.0822,\n", " 'rank_mean': 3.571,\n", " 'killed': '',\n", " 'wall_s': 5.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m12_K5_spread_s1234_346e68',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 46128,\n", " 'acc': 0.6176,\n", " 'ce_bits': 1.4456,\n", " 'cum_bits': 6.4058,\n", " 'delivered_bits': 6.5544,\n", " 'marginal_bits': '[2.5694, 1.3935, 0.8399, 0.3745, 0.3321, 0.2357, 0.1408, 0.1377, 0.1171, 0.104, 0.0638, 0.0973]',\n", " 'redundancy': 0.1374,\n", " 'cancellation': 0.4005,\n", " 'stage_snr': 0.4658,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 5.3,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m12_K5_spread_s5678_d04d82',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 46128,\n", " 'acc': 0.6133,\n", " 'ce_bits': 1.4703,\n", " 'cum_bits': 6.3452,\n", " 'delivered_bits': 6.5297,\n", " 'marginal_bits': '[2.2157, 1.6542, 0.8482, 0.41, 0.3625, 0.21, 0.1498, 0.1647, 0.0719, 0.1023, 0.0844, 0.0716]',\n", " 'redundancy': 0.16,\n", " 'cancellation': 0.4304,\n", " 'stage_snr': 0.4715,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 5.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m16_K4_free_s1234_c67f01',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 75264,\n", " 'acc': 0.6506,\n", " 'ce_bits': 1.3595,\n", " 'cum_bits': 6.4765,\n", " 'delivered_bits': 6.6405,\n", " 'marginal_bits': '[2.0315, 1.4738, 0.9752, 0.4626, 0.446, 0.2459, 0.18, 0.1154, 0.1039, 0.0992, 0.0979, 0.0775, 0.01, 0.0728, 0.067, 0.0176]',\n", " 'redundancy': 0.1852,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4698,\n", " 'dev_mean': 0.0932,\n", " 'rank_mean': 3.251,\n", " 'killed': '',\n", " 'wall_s': 8.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m16_K4_free_s5678_edce59',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 75264,\n", " 'acc': 0.66,\n", " 'ce_bits': 1.311,\n", " 'cum_bits': 6.4743,\n", " 'delivered_bits': 6.689,\n", " 'marginal_bits': '[1.9963, 1.4521, 0.9066, 0.5418, 0.3509, 0.2195, 0.2372, 0.1394, 0.1147, 0.1269, 0.1519, 0.0184, 0.088, 0.056, 0.0571, 0.0176]',\n", " 'redundancy': 0.1914,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4565,\n", " 'dev_mean': 0.1017,\n", " 'rank_mean': 3.411,\n", " 'killed': '',\n", " 'wall_s': 8.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m16_K4_spread_s1234_b01a9b',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 75264,\n", " 'acc': 0.6517,\n", " 'ce_bits': 1.3529,\n", " 'cum_bits': 6.4649,\n", " 'delivered_bits': 6.6471,\n", " 'marginal_bits': '[2.1028, 1.3648, 0.8906, 0.5949, 0.2653, 0.3467, 0.18, 0.1605, 0.1285, 0.1223, 0.081, 0.0577, 0.0462, 0.0339, 0.0523, 0.0375]',\n", " 'redundancy': 0.1737,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.5889,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 7.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'cross_m16_K4_spread_s5678_ee53da',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 75264,\n", " 'acc': 0.6632,\n", " 'ce_bits': 1.3185,\n", " 'cum_bits': 6.5295,\n", " 'delivered_bits': 6.6815,\n", " 'marginal_bits': '[2.0934, 1.3997, 0.8913, 0.4767, 0.3722, 0.2896, 0.2358, 0.1426, 0.1574, 0.0938, 0.1345, 0.0263, 0.0541, 0.0723, 0.0593, 0.0304]',\n", " 'redundancy': 0.1855,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.481,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 7.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/36 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8640,\n", " 'acc': 0.1588,\n", " 'ce_bits': 3.8523,\n", " 'cum_bits': 3.4673,\n", " 'delivered_bits': 4.1477,\n", " 'marginal_bits': '[3.4673]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3346,\n", " 'dev_mean': -0.1933,\n", " 'rank_mean': 3.456,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_free_s5678_800880',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8640,\n", " 'acc': 0.1642,\n", " 'ce_bits': 3.8242,\n", " 'cum_bits': 3.1941,\n", " 'delivered_bits': 4.1758,\n", " 'marginal_bits': '[3.1941]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2974,\n", " 'dev_mean': -0.0812,\n", " 'rank_mean': 3.744,\n", " 'killed': '',\n", " 'wall_s': 0.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_spread_s1234_5dc570',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8640,\n", " 'acc': 0.2213,\n", " 'ce_bits': 3.4842,\n", " 'cum_bits': 2.8509,\n", " 'delivered_bits': 4.5158,\n", " 'marginal_bits': '[2.8509]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2285,\n", " 'dev_mean': 0.0196,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_spread_s5678_338545',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 200,\n", " 'params': 8640,\n", " 'acc': 0.222,\n", " 'ce_bits': 3.5494,\n", " 'cum_bits': 2.7722,\n", " 'delivered_bits': 4.4506,\n", " 'marginal_bits': '[2.7722]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2526,\n", " 'dev_mean': 0.0199,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 0.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'cross_m12_K5_free_s1234_41ba1d',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 46128,\n", " 'acc': 0.719,\n", " 'ce_bits': 1.4086,\n", " 'cum_bits': 6.5719,\n", " 'delivered_bits': 6.5914,\n", " 'marginal_bits': '[2.4615, 1.32, 0.7649, 0.5384, 0.4133, 0.2699, 0.218, 0.1618, 0.1331, 0.1094, 0.0905, 0.091]',\n", " 'redundancy': 0.1695,\n", " 'cancellation': 0.3459,\n", " 'stage_snr': 0.4448,\n", " 'dev_mean': 0.1329,\n", " 'rank_mean': 3.676,\n", " 'killed': '',\n", " 'wall_s': 18.4,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m16_K4_free_s1234_c67f01',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 75264,\n", " 'acc': 0.7255,\n", " 'ce_bits': 1.5603,\n", " 'cum_bits': 6.6382,\n", " 'delivered_bits': 6.4397,\n", " 'marginal_bits': '[1.8261, 1.5147, 1.105, 0.5414, 0.4241, 0.2561, 0.2533, 0.1259, 0.1145, 0.0993, 0.0781, 0.0935, 0.0428, 0.0754, 0.0606, 0.0274]',\n", " 'redundancy': 0.2054,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3733,\n", " 'dev_mean': 0.1,\n", " 'rank_mean': 3.308,\n", " 'killed': '',\n", " 'wall_s': 29.6,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 8640,\n", " 'acc': 0.3052,\n", " 'ce_bits': 2.6917,\n", " 'cum_bits': 3.5431,\n", " 'delivered_bits': 5.3083,\n", " 'marginal_bits': '[3.5431]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3473,\n", " 'dev_mean': -0.1484,\n", " 'rank_mean': 3.586,\n", " 'killed': '',\n", " 'wall_s': 1.5,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_spread_s1234_5dc570',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 8640,\n", " 'acc': 0.3153,\n", " 'ce_bits': 2.6453,\n", " 'cum_bits': 2.9511,\n", " 'delivered_bits': 5.3547,\n", " 'marginal_bits': '[2.9511]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2547,\n", " 'dev_mean': 0.0196,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 1.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m12_K5_spread_s1234_1c84c6',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 10236,\n", " 'acc': 0.4701,\n", " 'ce_bits': 1.9447,\n", " 'cum_bits': 6.2946,\n", " 'delivered_bits': 6.0553,\n", " 'marginal_bits': '[2.6624, 1.3563, 0.8241, 0.4233, 0.3055, 0.1615, 0.1862, 0.1151, 0.0199, 0.121, 0.0471, 0.0724]',\n", " 'redundancy': 0.3236,\n", " 'cancellation': 0.3184,\n", " 'stage_snr': 0.436,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 10.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m16_K4_spread_s1234_34c9f7',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 11536,\n", " 'acc': 0.4141,\n", " 'ce_bits': 2.1494,\n", " 'cum_bits': 6.2518,\n", " 'delivered_bits': 5.8506,\n", " 'marginal_bits': '[2.223, 1.3927, 0.5685, 0.486, 0.4527, 0.1659, 0.2798, 0.1631, 0.0992, 0.1025, 0.0998, 0.0539, 0.0262, 0.099, 0.0, 0.0395]',\n", " 'redundancy': 0.3918,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4592,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 14.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m16_K4_spread_s1234_190427',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 15616,\n", " 'acc': 0.6963,\n", " 'ce_bits': 1.2771,\n", " 'cum_bits': 6.6301,\n", " 'delivered_bits': 6.7229,\n", " 'marginal_bits': '[2.4488, 1.2806, 0.7416, 0.5198, 0.3425, 0.2847, 0.2445, 0.1801, 0.1089, 0.1282, 0.0612, 0.0819, 0.0218, 0.091, 0.0306, 0.064]',\n", " 'redundancy': 0.1751,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3705,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 14.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'sum_m16_K4_free_s5678_a631de',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 11536,\n", " 'acc': 0.4229,\n", " 'ce_bits': 2.0866,\n", " 'cum_bits': 6.2979,\n", " 'delivered_bits': 5.9134,\n", " 'marginal_bits': '[2.4988, 1.3273, 0.7245, 0.4308, 0.3731, 0.1173, 0.1541, 0.1329, 0.1295, 0.1058, 0.052, 0.0604, 0.0814, 0.0364, 0.0738, 0.0]',\n", " 'redundancy': 0.4233,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.445,\n", " 'dev_mean': 0.2844,\n", " 'rank_mean': 3.845,\n", " 'killed': '',\n", " 'wall_s': 18.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'cross_m16_K4_spread_s1234_b01a9b',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 75264,\n", " 'acc': 0.7144,\n", " 'ce_bits': 1.599,\n", " 'cum_bits': 6.6568,\n", " 'delivered_bits': 6.401,\n", " 'marginal_bits': '[2.0986, 1.4003, 0.9186, 0.6137, 0.3039, 0.3379, 0.1957, 0.1544, 0.1484, 0.1231, 0.0715, 0.0768, 0.0503, 0.0399, 0.0533, 0.0706]',\n", " 'redundancy': 0.1899,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4642,\n", " 'dev_mean': 0.4644,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 24.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_spread_s5678_761442',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 15616,\n", " 'acc': 0.7004,\n", " 'ce_bits': 1.342,\n", " 'cum_bits': 6.6183,\n", " 'delivered_bits': 6.658,\n", " 'marginal_bits': '[2.3219, 1.3294, 0.8387, 0.551, 0.3483, 0.2429, 0.2211, 0.1367, 0.1006, 0.136, 0.1013, 0.0435, 0.077, 0.0853, 0.0514, 0.0332]',\n", " 'redundancy': 0.1754,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.359,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 14.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_free_s5678_d700ae',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 15616,\n", " 'acc': 0.7019,\n", " 'ce_bits': 1.2668,\n", " 'cum_bits': 6.6416,\n", " 'delivered_bits': 6.7332,\n", " 'marginal_bits': '[2.3748, 1.3549, 0.8591, 0.4468, 0.3895, 0.2311, 0.2017, 0.1274, 0.1685, 0.1055, 0.1192, 0.0015, 0.1274, 0.0474, 0.0695, 0.0176]',\n", " 'redundancy': 0.1896,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3153,\n", " 'dev_mean': 0.1757,\n", " 'rank_mean': 3.592,\n", " 'killed': '',\n", " 'wall_s': 16.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_spread_s5678_338545',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 8640,\n", " 'acc': 0.3156,\n", " 'ce_bits': 2.6624,\n", " 'cum_bits': 2.9003,\n", " 'delivered_bits': 5.3376,\n", " 'marginal_bits': '[2.9003]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2618,\n", " 'dev_mean': 0.0199,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 1.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m12_K5_free_s1234_d6fd74',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 10236,\n", " 'acc': 0.4637,\n", " 'ce_bits': 1.9609,\n", " 'cum_bits': 6.2838,\n", " 'delivered_bits': 6.0391,\n", " 'marginal_bits': '[2.421, 1.5628, 0.618, 0.4326, 0.4234, 0.3161, 0.1535, 0.1277, 0.0413, 0.0777, 0.0653, 0.0444]',\n", " 'redundancy': 0.3693,\n", " 'cancellation': 0.3563,\n", " 'stage_snr': 0.4021,\n", " 'dev_mean': 0.2074,\n", " 'rank_mean': 3.89,\n", " 'killed': '',\n", " 'wall_s': 12.9,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'cross_m12_K5_spread_s1234_346e68',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 46128,\n", " 'acc': 0.703,\n", " 'ce_bits': 1.4509,\n", " 'cum_bits': 6.5599,\n", " 'delivered_bits': 6.5491,\n", " 'marginal_bits': '[2.5788, 1.4416, 0.814, 0.4013, 0.3668, 0.2502, 0.169, 0.1109, 0.1101, 0.1362, 0.0816, 0.0994]',\n", " 'redundancy': 0.1612,\n", " 'cancellation': 0.3697,\n", " 'stage_snr': 0.3722,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 16.2,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'single_m1_K64_free_s5678_800880',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 8640,\n", " 'acc': 0.3262,\n", " 'ce_bits': 2.6444,\n", " 'cum_bits': 3.1654,\n", " 'delivered_bits': 5.3556,\n", " 'marginal_bits': '[3.1654]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2619,\n", " 'dev_mean': -0.0452,\n", " 'rank_mean': 3.834,\n", " 'killed': '',\n", " 'wall_s': 1.5,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m16_K4_free_s1234_52ba39',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 15616,\n", " 'acc': 0.7006,\n", " 'ce_bits': 1.2956,\n", " 'cum_bits': 6.62,\n", " 'delivered_bits': 6.7044,\n", " 'marginal_bits': '[2.484, 1.2902, 0.8344, 0.5119, 0.3798, 0.2305, 0.194, 0.1542, 0.0944, 0.1267, 0.097, 0.0303, 0.055, 0.0627, 0.0074, 0.0674]',\n", " 'redundancy': 0.1807,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.2867,\n", " 'dev_mean': 0.1643,\n", " 'rank_mean': 3.56,\n", " 'killed': '',\n", " 'wall_s': 16.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'sum_m16_K4_free_s1234_85541a',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 11536,\n", " 'acc': 0.4779,\n", " 'ce_bits': 1.9601,\n", " 'cum_bits': 6.3847,\n", " 'delivered_bits': 6.0399,\n", " 'marginal_bits': '[2.0197, 1.5604, 0.9234, 0.549, 0.3496, 0.2922, 0.1365, 0.1184, 0.0822, 0.1053, 0.0582, 0.0479, 0.0314, 0.0397, 0.0348, 0.0361]',\n", " 'redundancy': 0.3758,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4205,\n", " 'dev_mean': 0.2947,\n", " 'rank_mean': 3.858,\n", " 'killed': '',\n", " 'wall_s': 17.1,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m12_K5_free_s5678_f1be97',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 10236,\n", " 'acc': 0.479,\n", " 'ce_bits': 1.9821,\n", " 'cum_bits': 6.2733,\n", " 'delivered_bits': 6.0179,\n", " 'marginal_bits': '[2.3669, 1.7378, 0.6613, 0.4194, 0.2719, 0.197, 0.1508, 0.1933, 0.0988, 0.0767, 0.0792, 0.0202]',\n", " 'redundancy': 0.3604,\n", " 'cancellation': 0.3763,\n", " 'stage_snr': 0.3591,\n", " 'dev_mean': 0.1909,\n", " 'rank_mean': 3.854,\n", " 'killed': '',\n", " 'wall_s': 12.9,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'cross_m16_K4_spread_s5678_ee53da',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 75264,\n", " 'acc': 0.7234,\n", " 'ce_bits': 1.5585,\n", " 'cum_bits': 6.672,\n", " 'delivered_bits': 6.4415,\n", " 'marginal_bits': '[2.0578, 1.3765, 1.0057, 0.5062, 0.3913, 0.2875, 0.2272, 0.1739, 0.1436, 0.0924, 0.1493, 0.0513, 0.0645, 0.0641, 0.0682, 0.0126]',\n", " 'redundancy': 0.1968,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.387,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 24.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'},\n", " {'arm_id': 'cross_m12_K5_spread_s5678_d04d82',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 46128,\n", " 'acc': 0.7117,\n", " 'ce_bits': 1.4729,\n", " 'cum_bits': 6.522,\n", " 'delivered_bits': 6.5271,\n", " 'marginal_bits': '[2.2159, 1.607, 0.8728, 0.4463, 0.3923, 0.2763, 0.1616, 0.1745, 0.103, 0.121, 0.0938, 0.0575]',\n", " 'redundancy': 0.1755,\n", " 'cancellation': 0.419,\n", " 'stage_snr': 0.361,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 16.2,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m12_K5_spread_s5678_a6f56c',\n", " 'op': 'sum',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 10236,\n", " 'acc': 0.4636,\n", " 'ce_bits': 2.0173,\n", " 'cum_bits': 6.2281,\n", " 'delivered_bits': 5.9827,\n", " 'marginal_bits': '[2.4996, 1.5549, 0.7027, 0.4346, 0.2568, 0.1145, 0.1756, 0.1241, 0.1397, 0.0995, 0.0719, 0.0542]',\n", " 'redundancy': 0.3351,\n", " 'cancellation': 0.4297,\n", " 'stage_snr': 0.4187,\n", " 'dev_mean': 0.2146,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 10.9,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m16_K4_spread_s5678_179916',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'spread',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 11536,\n", " 'acc': 0.4467,\n", " 'ce_bits': 2.0385,\n", " 'cum_bits': 6.3263,\n", " 'delivered_bits': 5.9615,\n", " 'marginal_bits': '[2.5259, 1.2685, 0.7107, 0.626, 0.2913, 0.165, 0.1977, 0.094, 0.0907, 0.0864, 0.0803, 0.0305, 0.0606, 0.0053, 0.0798, 0.0135]',\n", " 'redundancy': 0.3628,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.4536,\n", " 'dev_mean': 0.4645,\n", " 'rank_mean': 4.0,\n", " 'killed': '',\n", " 'wall_s': 15.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'cross_m12_K5_free_s5678_d62bfc',\n", " 'op': 'cross',\n", " 'm': 12,\n", " 'K': 5,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 46128,\n", " 'acc': 0.7151,\n", " 'ce_bits': 1.4292,\n", " 'cum_bits': 6.5222,\n", " 'delivered_bits': 6.5708,\n", " 'marginal_bits': '[2.123, 1.7879, 0.9346, 0.4333, 0.3286, 0.2262, 0.1872, 0.1432, 0.1253, 0.117, 0.0915, 0.0244]',\n", " 'redundancy': 0.175,\n", " 'cancellation': 0.3584,\n", " 'stage_snr': 0.3985,\n", " 'dev_mean': 0.0848,\n", " 'rank_mean': 3.577,\n", " 'killed': '',\n", " 'wall_s': 19.3,\n", " 'rung': 1,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'cross_m16_K4_free_s5678_edce59',\n", " 'op': 'cross',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 75264,\n", " 'acc': 0.7246,\n", " 'ce_bits': 1.614,\n", " 'cum_bits': 6.6055,\n", " 'delivered_bits': 6.386,\n", " 'marginal_bits': '[1.9679, 1.373, 1.0335, 0.5241, 0.4089, 0.2683, 0.2176, 0.1843, 0.1192, 0.1241, 0.1305, 0.025, 0.1026, 0.0472, 0.0696, 0.0097]',\n", " 'redundancy': 0.2082,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.3596,\n", " 'dev_mean': 0.1114,\n", " 'rank_mean': 3.437,\n", " 'killed': '',\n", " 'wall_s': 28.6,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/24 by cum_bits'}]" ] }, "metadata": {}, "execution_count": 5 } ] }, { "cell_type": "markdown", "source": [ "# aleph lm" ], "metadata": { "id": "lCbNIxga5U0u" } }, { "cell_type": "code", "source": [ "# lexical_atlas.py\n", "\"\"\"\n", "Lexical Atlas — the full wordnet-lexical-topology vocabulary on the sphere\n", "===========================================================================\n", "\n", "Extracts the ENTIRE AbstractPhil/wordnet-lexical-topology setup (~12.8M\n", "n-grams across {nltk, hf, unicode} x {char, word} x {1..5}gram configs) into\n", "spherical coordinates, correctly spaced — where \"correct\" is determined by\n", "capacity mathematics, not hope.\n", "\n", "THE CAPACITY LAW (computed exactly, 2026-06-09):\n", " 12.8M uniformly spaced points on S^(D-1), median nearest-neighbor angle:\n", " D=4 : 0.363 deg -> 0.06 logits of address contrast at tau=0.1\n", " (neighbors indistinguishable through K=64; fp16\n", " cannot resolve the cosines, fp32 marginal)\n", " D=32: 39.1 deg | D=48: 47.6 deg -> 7-8 logits, comfortable\n", " The CM-band result (band-valid D=32-112, sweet spot 32-56) independently\n", " prescribes the same range. THEREFORE the atlas is TWO-TIER:\n", "\n", " TIER 1 (base) : deterministic low-discrepancy placement at band-valid D\n", " (default 48) — scrambled-Sobol -> Gaussian -> normalize.\n", " Uniform by construction, reproducible by seed, unique\n", " per n-gram. This is \"spaced on the sphere correctly.\"\n", " TIER 2 (view) : the LEARNED D_addr=4 address-space view extracted from a\n", " trained AlephLM checkpoint — per n-gram: bytes -> pad ->\n", " trigrams -> kappa rows (W_kappa o byte_emb) -> mean ->\n", " normalize. This is where the model actually PLACED the\n", " vocabulary; crowded by necessity (see law), meaningful\n", " as geometry-of-content, not as unique identity.\n", "\n", "Honesty on the learned view: mean composition is order-insensitive, so\n", "anagrammatic n-grams (same trigram multiset) COLLIDE; collisions are counted\n", "and reported per config. The deterministic tier never collides.\n", "\n", "Per-config outputs:\n", " atlas/{config}.parquet columns: ngram, rank, frequency, n_tri,\n", " vec_base (D_base floats), vec_view (4 floats)\n", " atlas/{config}.stats.json NN-angle distribution (sampled), statute of\n", " both tiers (4k subsample), collision count\n", "\n", "Usage:\n", " python lexical_atlas.py --checkpoint aleph_lm_hybrid_corpus.pt \\\\\n", " --configs char_eng_unigram char_eng_2gram char_eng_3gram \\\\\n", " char_eng_4gram char_eng_5gram --d-base 48\n", " # --configs all -> every config in the dataset (~12.8M rows total)\n", "\n", "Depends: aleph_lm.py (+ its deps), pyarrow, huggingface_hub.\n", "Author: AbstractPhil + Mirel Date: 2026-06-09 License: MIT\n", "\"\"\"\n", "\n", "from __future__ import annotations\n", "\n", "import json\n", "import math\n", "import os\n", "from dataclasses import dataclass, field\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import numpy as np\n", "import torch\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from aleph_trigram_lm import statute\n", "except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", "\n", "DATASET = \"AbstractPhil/wordnet-lexical-topology\"\n", "PAD_BYTE = 0x00 # reserved pad symbol (documented, learned slot)\n", "\n", "_GRAMS = (\"unigram\", \"2gram\", \"3gram\", \"4gram\", \"5gram\")\n", "SOURCE_CONFIGS = ([f\"nltk_{k}_eng_{n}\" for k in (\"char\", \"word\") for n in _GRAMS]\n", " + [f\"hf_{k}_eng_{n}\" for k in (\"char\", \"word\") for n in _GRAMS]\n", " + [f\"unicode_global_{n}\" for n in _GRAMS])\n", "LEGACY_CONFIGS = [f\"{k}_eng_{n}\" for k in (\"char\", \"word\") for n in _GRAMS]\n", "# legacy unprefixed configs are pre-merged ANCESTORS (verified: char_eng_3gram\n", "# is a superset of nltk_char_eng_3gram, freq corr 0.914) — excluded from 'all'\n", "# to avoid double counting; available explicitly.\n", "ALL_CONFIGS = SOURCE_CONFIGS\n", "\n", "\n", "def source_of(config: str) -> str:\n", " for s in (\"nltk\", \"hf\", \"unicode\"):\n", " if config.startswith(s):\n", " return s\n", " return \"legacy\"\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Config\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class AtlasConfig:\n", " checkpoint: Optional[str] = None # AlephLM ckpt (None = base tier only)\n", " configs: List[str] = field(default_factory=lambda: [\n", " \"char_eng_unigram\", \"char_eng_2gram\", \"char_eng_3gram\",\n", " \"char_eng_4gram\", \"char_eng_5gram\"])\n", " d_base: int = 48 # band-valid (CM sweet spot 32-56)\n", " base_seed: int = 1234 # determinism of Tier 1\n", " out_dir: str = \"atlas\"\n", " batch: int = 65536\n", " max_tri: int = 16 # n-grams longer than 48 bytes truncated\n", " stats_sample: int = 4000\n", " device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier 1 — deterministic band-valid base (correct spacing by construction)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class SobolSphere:\n", " \"\"\"Low-discrepancy points on S^(D-1): scrambled Sobol -> inverse-normal ->\n", " normalize. Deterministic per (seed, global index): the same n-gram (by its\n", " global rank position) always receives the same point. Never collides.\"\"\"\n", "\n", " def __init__(self, D: int, seed: int):\n", " self.D, self.seed = D, seed\n", " self.eng = torch.quasirandom.SobolEngine(D, scramble=True, seed=seed)\n", " self._cursor = 0\n", "\n", " def take(self, n: int) -> Tensor:\n", " u = self.eng.draw(n).clamp(1e-6, 1 - 1e-6)\n", " g = torch.erfinv(2 * u - 1) * math.sqrt(2.0) # inverse normal CDF\n", " self._cursor += n\n", " return F.normalize(g, dim=-1)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier 2 — learned address-space view (the model's own placement)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class LearnedView:\n", " \"\"\"kappa-row composer from a trained AlephLM checkpoint.\"\"\"\n", "\n", " def __init__(self, checkpoint: str, device: str):\n", " try:\n", " from aleph_lm import AlephLM, AlephLMConfig\n", " except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", " d = torch.load(checkpoint, map_location=device, weights_only=False)\n", " fields = AlephLMConfig.__dataclass_fields__\n", " cfg = AlephLMConfig(**{k: v for k, v in d[\"config\"].items() if k in fields})\n", " bank = d.get(\"bank\", None)\n", " self.model = AlephLM(cfg, bank=bank).to(device)\n", " self.model.load_state_dict(d[\"model_state_dict\"])\n", " self.model.eval()\n", " self.cfg, self.device = cfg, device\n", "\n", " @torch.no_grad()\n", " def compose(self, tri: Tensor, n_tri: Tensor) -> Tensor:\n", " \"\"\"tri: (B, T, 3) padded trigram bytes; n_tri: (B,) valid counts.\n", " Returns (B, D_addr) unit rows: normalized mean of per-trigram\n", " kappa rows over the valid prefix. Order-insensitive (collisions\n", " among anagrams; counted upstream).\"\"\"\n", " m = self.model\n", " tri = tri.to(self.device)\n", " e = sum(emb(tri[..., i]) for i, emb in enumerate(m.byte_emb)) # (B,T,d)\n", " rows = F.normalize(m.W_kappa(e), dim=-1) # (B,T,Da)\n", " mask = (torch.arange(tri.shape[1], device=self.device)[None, :]\n", " < n_tri.to(self.device)[:, None]).float().unsqueeze(-1)\n", " mean = (rows * mask).sum(1) / mask.sum(1).clamp_min(1e-9)\n", " return F.normalize(mean, dim=-1).cpu()\n", "\n", "\n", "def ngrams_to_trigrams(ngrams: List[str], max_tri: int\n", " ) -> Tuple[Tensor, Tensor, np.ndarray]:\n", " \"\"\"UTF-8 encode, pad to multiple of 3 with PAD_BYTE, frame as trigrams.\n", " Returns (B, max_tri, 3) bytes, (B,) counts, and the trigram-multiset hash\n", " per n-gram (for anagram-collision counting).\"\"\"\n", " B = len(ngrams)\n", " out = np.zeros((B, max_tri, 3), dtype=np.int64)\n", " counts = np.zeros(B, dtype=np.int64)\n", " mhash = np.zeros(B, dtype=np.uint64)\n", " for i, s in enumerate(ngrams):\n", " b = str(s).encode(\"utf-8\", errors=\"ignore\")[: 3 * max_tri]\n", " if len(b) % 3:\n", " b = b + bytes([PAD_BYTE]) * (3 - len(b) % 3)\n", " t = np.frombuffer(b, dtype=np.uint8).reshape(-1, 3).astype(np.int64)\n", " n = len(t)\n", " out[i, :n] = t\n", " counts[i] = max(n, 1)\n", " ids = (t[:, 0] * 65536 + t[:, 1] * 256 + t[:, 2]).astype(np.uint64)\n", " h = np.uint64(0)\n", " for v in np.sort(ids): # order-free multiset hash\n", " h = (h * np.uint64(1099511628211)) ^ (v + np.uint64(0x9E3779B9))\n", " mhash[i] = h ^ np.uint64(n)\n", " return torch.from_numpy(out), torch.from_numpy(counts), mhash\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Spacing battery\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def spacing_stats(vecs: Tensor, sample: int, seed: int = 0) -> Dict:\n", " \"\"\"Sampled NN-angle distribution + statute on a subsample.\"\"\"\n", " g = torch.Generator().manual_seed(seed)\n", " idx = torch.randperm(len(vecs), generator=g)[: min(sample, len(vecs))]\n", " X = F.normalize(vecs[idx].float(), dim=-1)\n", " cos = (X @ X.t()).clamp(-1, 1)\n", " cos.fill_diagonal_(-1)\n", " nn_deg = torch.acos(cos.max(dim=-1).values) * 180 / math.pi\n", " st = statute(X)\n", " return {\"nn_deg_median\": nn_deg.median().item(),\n", " \"nn_deg_p05\": nn_deg.quantile(0.05).item(),\n", " \"nn_deg_p95\": nn_deg.quantile(0.95).item(),\n", " \"statute\": st}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Extraction\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def extract_config(name: str, cfg: AtlasConfig, sobol: SobolSphere,\n", " view: Optional[LearnedView]) -> Dict:\n", " import pyarrow as pa\n", " import pyarrow.parquet as pq\n", " from huggingface_hub import hf_hub_download\n", "\n", " path = hf_hub_download(DATASET, f\"data/{name}-00000-of-00001.parquet\",\n", " repo_type=\"dataset\")\n", " t = pq.read_table(path, columns=[\"ngram\", \"rank\", \"frequency\"]) \\\n", " .to_pandas().sort_values(\"rank\").reset_index(drop=True)\n", " N = len(t)\n", " print(f\"[{name}] {N:,} n-grams\")\n", "\n", " base = sobol.take(N) # (N, D_base)\n", " views, counts, hashes = [], [], []\n", " if view is not None:\n", " for s0 in range(0, N, cfg.batch):\n", " chunk = t[\"ngram\"].iloc[s0: s0 + cfg.batch].tolist()\n", " tri, n_tri, mh = ngrams_to_trigrams(chunk, cfg.max_tri)\n", " views.append(view.compose(tri, n_tri))\n", " counts.append(n_tri)\n", " hashes.append(mh)\n", " vview = torch.cat(views)\n", " n_tri = torch.cat(counts)\n", " mh = np.concatenate(hashes)\n", " n_coll = int(N - len(np.unique(mh)))\n", " else:\n", " vview, n_tri, n_coll = None, None, 0\n", "\n", " os.makedirs(cfg.out_dir, exist_ok=True)\n", " cols = {\"ngram\": pa.array(t[\"ngram\"].astype(str)),\n", " \"rank\": pa.array(t[\"rank\"].astype(\"int64\")),\n", " \"frequency\": pa.array(t[\"frequency\"].astype(\"float64\")),\n", " \"vec_base\": pa.array(base.numpy().tolist(),\n", " type=pa.list_(pa.float32(), cfg.d_base))}\n", " if vview is not None:\n", " cols[\"n_tri\"] = pa.array(n_tri.numpy().astype(\"int8\"))\n", " cols[\"vec_view\"] = pa.array(vview.numpy().tolist(),\n", " type=pa.list_(pa.float32(), vview.shape[1]))\n", " out_path = os.path.join(cfg.out_dir, f\"{name}.parquet\")\n", " pq.write_table(pa.table(cols), out_path)\n", "\n", " stats = {\"config\": name, \"n\": N, \"d_base\": cfg.d_base,\n", " \"anagram_collisions_view\": n_coll,\n", " \"base\": spacing_stats(base, cfg.stats_sample)}\n", " if vview is not None:\n", " stats[\"view\"] = spacing_stats(vview, cfg.stats_sample)\n", " with open(os.path.join(cfg.out_dir, f\"{name}.stats.json\"), \"w\") as f:\n", " json.dump(stats, f, indent=2, default=str)\n", " print(f\" base NN {stats['base']['nn_deg_median']:.2f} deg \"\n", " f\"(statute {stats['base']['statute']['statute']})\"\n", " + (f\" view NN {stats['view']['nn_deg_median']:.3f} deg \"\n", " f\"(statute {stats['view']['statute']['statute']}, \"\n", " f\"collisions {n_coll})\" if vview is not None else \"\")\n", " + f\" -> {out_path}\")\n", " return stats\n", "\n", "\n", "def build_atlas(cfg: AtlasConfig) -> List[Dict]:\n", " names = ALL_CONFIGS if cfg.configs == [\"all\"] else cfg.configs\n", " sobol = SobolSphere(cfg.d_base, cfg.base_seed) # ONE stream:\n", " view = LearnedView(cfg.checkpoint, cfg.device) if cfg.checkpoint else None\n", " # global index = unique placement across ALL configs (never reused)\n", " all_stats = []\n", " for name in names:\n", " all_stats.append(extract_config(name, cfg, sobol, view))\n", " total = sum(s[\"n\"] for s in all_stats)\n", " print(f\"\\n[atlas] {total:,} n-grams placed at D={cfg.d_base} \"\n", " f\"(Tier 1, deterministic, collision-free)\"\n", " + (f\" + learned D=4 view (Tier 2)\" if view else \"\"))\n", " return all_stats\n", "\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Canon — weighted dedupe across sources: ONE STRING, ONE POINT\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Cross-config duplicates of the same n-gram must not receive different\n", "# Tier-1 placements. Canonization: per-config frequencies are normalized\n", "# (sum to 1 within config — scale-free across sources), scaled by a\n", "# per-source weight (HF elevated: frequency-weighted definitions with\n", "# cardinality), summed per unique string, re-ranked, and placed once.\n", "\n", "DEFAULT_SOURCE_WEIGHTS = {\"hf\": 5.0, \"nltk\": 1.0, \"unicode\": 1.0, \"legacy\": 0.0}\n", "\n", "\n", "def canonize(cfg: AtlasConfig,\n", " source_weights: Optional[Dict[str, float]] = None,\n", " configs: Optional[List[str]] = None) -> Dict:\n", " \"\"\"Build the canonical deduplicated atlas directly from the dataset.\"\"\"\n", " import pandas as pd\n", " import pyarrow as pa\n", " import pyarrow.parquet as pq\n", " from huggingface_hub import hf_hub_download\n", "\n", " W = dict(DEFAULT_SOURCE_WEIGHTS)\n", " if source_weights:\n", " W.update(source_weights)\n", " names = configs or SOURCE_CONFIGS\n", "\n", " frames, prov = [], []\n", " for name in names:\n", " lam = W.get(source_of(name), 0.0)\n", " if lam <= 0:\n", " print(f\"[canon] {name}: weight 0 — skipped\")\n", " continue\n", " p = hf_hub_download(DATASET, f\"data/{name}-00000-of-00001.parquet\",\n", " repo_type=\"dataset\")\n", " t = pq.read_table(p, columns=[\"ngram\", \"frequency\"]).to_pandas()\n", " t[\"ngram\"] = t[\"ngram\"].astype(str)\n", " t[\"w\"] = lam * t[\"frequency\"] / max(t[\"frequency\"].sum(), 1e-30)\n", " t[\"src\"] = source_of(name)\n", " frames.append(t[[\"ngram\", \"w\", \"src\"]])\n", " print(f\"[canon] {name}: {len(t):,} rows (lambda={lam})\")\n", " allrows = pd.concat(frames, ignore_index=True)\n", " print(f\"[canon] total rows {len(allrows):,}\")\n", "\n", " agg = allrows.groupby(\"ngram\", sort=False).agg(\n", " weight=(\"w\", \"sum\"),\n", " n_sources=(\"src\", \"nunique\"),\n", " sources=(\"src\", lambda s: \"+\".join(sorted(set(s)))))\n", " agg = agg.sort_values(\"weight\", ascending=False).reset_index()\n", " N = len(agg)\n", " dup = len(allrows) - N\n", " print(f\"[canon] unique n-grams {N:,} (merged {dup:,} duplicate rows)\")\n", "\n", " # Tier 1: one fresh stream over the canonical ranking — one string, one point\n", " sobol = SobolSphere(cfg.d_base, cfg.base_seed)\n", " base = sobol.take(N)\n", "\n", " # Tier 2: learned view regenerated per unique string (pure function)\n", " view = LearnedView(cfg.checkpoint, cfg.device) if cfg.checkpoint else None\n", " vview, n_tri_all, n_coll = None, None, 0\n", " if view is not None:\n", " views, counts, hashes = [], [], []\n", " for s0 in range(0, N, cfg.batch):\n", " chunk = agg[\"ngram\"].iloc[s0: s0 + cfg.batch].tolist()\n", " tri, n_tri, mh = ngrams_to_trigrams(chunk, cfg.max_tri)\n", " views.append(view.compose(tri, n_tri))\n", " counts.append(n_tri)\n", " hashes.append(mh)\n", " vview = torch.cat(views)\n", " n_tri_all = torch.cat(counts)\n", " mh = np.concatenate(hashes)\n", " n_coll = int(N - len(np.unique(mh)))\n", "\n", " os.makedirs(cfg.out_dir, exist_ok=True)\n", " cols = {\"ngram\": pa.array(agg[\"ngram\"]),\n", " \"weight\": pa.array(agg[\"weight\"].astype(\"float64\")),\n", " \"rank\": pa.array(np.arange(1, N + 1, dtype=np.int64)),\n", " \"n_sources\": pa.array(agg[\"n_sources\"].astype(\"int8\")),\n", " \"sources\": pa.array(agg[\"sources\"]),\n", " \"vec_base\": pa.array(base.numpy().tolist(),\n", " type=pa.list_(pa.float32(), cfg.d_base))}\n", " if vview is not None:\n", " cols[\"n_tri\"] = pa.array(n_tri_all.numpy().astype(\"int8\"))\n", " cols[\"vec_view\"] = pa.array(vview.numpy().tolist(),\n", " type=pa.list_(pa.float32(), vview.shape[1]))\n", " out_path = os.path.join(cfg.out_dir, \"canon.parquet\")\n", " pq.write_table(pa.table(cols), out_path)\n", "\n", " stats = {\"unique\": N, \"merged_duplicates\": dup,\n", " \"source_weights\": W, \"configs\": names,\n", " \"anagram_collisions_view\": n_coll,\n", " \"base\": spacing_stats(base, cfg.stats_sample)}\n", " if vview is not None:\n", " stats[\"view\"] = spacing_stats(vview, cfg.stats_sample)\n", " with open(os.path.join(cfg.out_dir, \"canon.stats.json\"), \"w\") as f:\n", " json.dump(stats, f, indent=2, default=str)\n", " print(f\"[canon] -> {out_path} \"\n", " f\"(base NN {stats['base']['nn_deg_median']:.2f} deg\"\n", " + (f\", view collisions {n_coll}\" if vview is not None else \"\") + \")\")\n", " return stats\n", "\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Stratified bank — round-robin across the granularity ladder\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Breadth-first sampling: rank-1 of every config, then rank-2, ... with\n", "# weighted dedupe along the way, until `target` unique n-grams. Yields a\n", "# compact multi-granularity candidate vocabulary stratified across\n", "# {source} x {char, word} x {1..5}gram. Two outputs:\n", "# bank_{target}.parquet the full multi-granularity bank\n", "# bank_{target}_tri.pt the 3-byte-exact subset as an (M, 3) tensor —\n", "# a DROP-IN AlephLM trigram bank (only exact\n", "# 3-byte strings can match raw next-trigram\n", "# targets; variable-length candidates await the\n", "# span-prediction head — v2, noted in log)\n", "\n", "def stratified_bank(cfg: AtlasConfig, target: int = 4096,\n", " source_weights: Optional[Dict[str, float]] = None,\n", " configs: Optional[List[str]] = None) -> Dict:\n", " import pandas as pd\n", " import pyarrow as pa\n", " import pyarrow.parquet as pq\n", " from huggingface_hub import hf_hub_download\n", "\n", " W = dict(DEFAULT_SOURCE_WEIGHTS)\n", " if source_weights:\n", " W.update(source_weights)\n", " names = [n for n in (configs or SOURCE_CONFIGS)\n", " if W.get(source_of(n), 0.0) > 0]\n", "\n", " tables = []\n", " for name in names:\n", " p = hf_hub_download(DATASET, f\"data/{name}-00000-of-00001.parquet\",\n", " repo_type=\"dataset\")\n", " t = pq.read_table(p, columns=[\"ngram\", \"rank\", \"frequency\"]).to_pandas()\n", " t[\"ngram\"] = t[\"ngram\"].astype(str)\n", " lam = W[source_of(name)]\n", " t[\"w\"] = lam * t[\"frequency\"] / max(t[\"frequency\"].sum(), 1e-30)\n", " t[\"config\"] = name\n", " tables.append(t.sort_values(\"rank\").reset_index(drop=True))\n", "\n", " chosen: Dict[str, Dict] = {}\n", " depth = 0\n", " while len(chosen) < target:\n", " progressed = False\n", " for t in tables:\n", " if depth >= len(t):\n", " continue\n", " progressed = True\n", " row = t.iloc[depth]\n", " rec = chosen.get(row.ngram)\n", " if rec is None:\n", " chosen[row.ngram] = {\"weight\": row.w, \"configs\": {row.config},\n", " \"first_depth\": depth}\n", " else:\n", " rec[\"weight\"] += row.w\n", " rec[\"configs\"].add(row.config)\n", " if len(chosen) >= target:\n", " break\n", " depth += 1\n", " if not progressed:\n", " break\n", " rows = [{\"ngram\": k, \"weight\": v[\"weight\"],\n", " \"n_configs\": len(v[\"configs\"]),\n", " \"configs\": \"+\".join(sorted(v[\"configs\"])),\n", " \"first_depth\": v[\"first_depth\"],\n", " \"n_bytes\": len(k.encode(\"utf-8\", errors=\"ignore\"))}\n", " for k, v in chosen.items()]\n", " bank = pd.DataFrame(rows).sort_values(\n", " [\"first_depth\", \"weight\"], ascending=[True, False]).reset_index(drop=True)\n", "\n", " os.makedirs(cfg.out_dir, exist_ok=True)\n", " out_pq = os.path.join(cfg.out_dir, f\"bank_{target}.parquet\")\n", " pq.write_table(pa.Table.from_pandas(bank, preserve_index=False), out_pq)\n", "\n", " tri_rows = [list(k.encode(\"utf-8\")) for k in bank[\"ngram\"]\n", " if len(k.encode(\"utf-8\", errors=\"ignore\")) == 3]\n", " tri = torch.tensor(tri_rows, dtype=torch.long) if tri_rows else torch.empty(0, 3, dtype=torch.long)\n", " out_pt = os.path.join(cfg.out_dir, f\"bank_{target}_tri.pt\")\n", " torch.save({\"bank\": tri, \"source\": \"stratified_atlas\",\n", " \"target\": target, \"weights\": W, \"configs\": names}, out_pt)\n", "\n", " comp = bank.groupby(bank[\"configs\"].str.split(\"+\").str[0]).size().to_dict()\n", " print(f\"[bank] {len(bank):,} unique n-grams at round-robin depth {depth}\"\n", " f\" (3-byte-exact: {len(tri):,} -> {out_pt})\")\n", " print(f\"[bank] multi-config members: {(bank.n_configs > 1).sum():,}\"\n", " f\" byte-length histogram: \"\n", " f\"{bank.n_bytes.value_counts().sort_index().to_dict()}\")\n", " print(f\"[bank] -> {out_pq}\")\n", " return {\"n\": len(bank), \"depth\": depth, \"n_tri\": len(tri),\n", " \"paths\": [out_pq, out_pt]}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Activation\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _running_in_notebook() -> bool:\n", " \"\"\"Pasted Colab/Jupyter cells run with __name__ == '__main__'; this keeps\n", " the demo/CLI below inert there so pasting never auto-fires it.\"\"\"\n", " try:\n", " from IPython import get_ipython\n", " return get_ipython() is not None\n", " except Exception:\n", " return False\n", "\n", "\n", "if __name__ == \"__main__\" and not _running_in_notebook():\n", " import argparse\n", " ap = argparse.ArgumentParser(description=\"Full lexical-topology atlas\")\n", " ap.add_argument(\"--checkpoint\", default=None)\n", " ap.add_argument(\"--configs\", nargs=\"+\",\n", " default=[\"char_eng_unigram\", \"char_eng_2gram\",\n", " \"char_eng_3gram\", \"char_eng_4gram\",\n", " \"char_eng_5gram\"])\n", " ap.add_argument(\"--d-base\", type=int, default=48)\n", " ap.add_argument(\"--out\", default=\"atlas\")\n", " ap.add_argument(\"--device\",\n", " default=\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " ap.add_argument(\"--canon\", action=\"store_true\",\n", " help=\"weighted dedupe across sources: one string, one point\")\n", " ap.add_argument(\"--weights\", default=\"hf=5,nltk=1,unicode=1,legacy=0\",\n", " help=\"per-source lambdas, e.g. hf=5,nltk=1,unicode=1\")\n", " ap.add_argument(\"--bank\", type=int, default=0,\n", " help=\"build a stratified bank of this many unique n-grams\")\n", " args, _unknown = ap.parse_known_args()\n", " acfg = AtlasConfig(checkpoint=args.checkpoint, configs=args.configs,\n", " d_base=args.d_base, out_dir=args.out, device=args.device)\n", " sw = {k: float(v) for k, v in\n", " (kv.split(\"=\") for kv in args.weights.split(\",\"))}\n", " if args.bank:\n", " stratified_bank(acfg, target=args.bank, source_weights=sw,\n", " configs=None if args.configs in ([\"all\"], [\"sources\"])\n", " else args.configs)\n", " elif args.canon:\n", " canonize(acfg, source_weights=sw,\n", " configs=None if args.configs == [\"all\"] else\n", " (SOURCE_CONFIGS if args.configs == [\"sources\"] else args.configs))\n", " else:\n", " build_atlas(acfg)" ], "metadata": { "id": "wyHZxZLc5V_Y" }, "execution_count": 4, "outputs": [] }, { "cell_type": "code", "source": [ "# aleph_routed_attention.py\n", "\"\"\"\n", "Aleph-Routed Attention — routing attention through a learned projective codebook\n", "=================================================================================\n", "\n", "Two variants of attention whose routing medium is the aleph signed-projective\n", "address (geolip-svae aleph_model.py lineage):\n", "\n", " HUB : linear attention whose feature map IS the aleph address.\n", " score(i,j) = over 2K oriented axes [+A; -A].\n", " Factors through two K-wide memories (the antipodal closed-form trick:\n", " the 2K tensor is never materialized). O(n*K*d), PURE GEMM — no gathers.\n", " Denominator is a dot product of strictly positive distributions, so it\n", " cannot vanish or flip sign (structurally stabler than elu+1 feature maps).\n", " Attention-matrix rank is bounded by 2K: K is the bandwidth knob,\n", " tau is the hardness knob.\n", "\n", " BUCKET : hard address. Each token's winner oriented half-axis is its bucket;\n", " exact softmax attention within sorted equal-width blocks (Reformer-style\n", " sort-and-window), masked to same-bucket pairs. One gather-bound mode for\n", " the A/B against the GEMM mode. Codebook receives gradient through a\n", " differentiable address-agreement bias added to the scores (hard argmax\n", " alone is gradient-dead w.r.t. the codebook).\n", "\n", "Shared geometric discipline (geolip-svae invariants honored):\n", " - q/k address rows are sphere-normalized onto S^(D_addr-1) (geometric premise)\n", " - nn.init.orthogonal_ on the address projections (load-bearing)\n", " - no BatchNorm, no Dropout on the geometric path, no GAP\n", " - codebook init: 'random' | 'fibonacci' (super-Fibonacci S^3 at D=4) | (K,D) array\n", " — 'custom' array supports TRANSPLANTING a trained AlephModel codebook.\n", "\n", "Preregistered basin test (decide before running):\n", " Train the routing codebook from scratch on a sequence task, then run the\n", " geolip-svae antipodal-collapse extraction on export_codebook().\n", " CLEAN (|deviation| < 0.05 on RP^(D-1)) -> cross-objective attractor evidence.\n", " DIRTY -> the attractor is reconstruction-specific.\n", " Either answer is data.\n", "\n", "Compile discipline (Phil's rule): forward() returns a single Tensor. All\n", "diagnostics (perplexity, margin, bucket load, confidence) live in the separate\n", "no-grad address_stats() method — never in the compiled hot path.\n", "\n", "Prior-art honesty for the writeup: hub is the linear-transformer/Performer\n", "family (kernel feature maps) crossed with Set-Transformer inducing points;\n", "bucket rhymes with Reformer/Routing Transformer. Novel content: the signed\n", "antipodal closed form as feature map, spherical D-space addresses, codebook\n", "transplant from reconstruction alephs, and the attractor test.\n", "\n", "Author: AbstractPhil + Mirel\n", "Date: 2026-06-09\n", "License: MIT\n", "\"\"\"\n", "\n", "from __future__ import annotations\n", "\n", "import math\n", "from dataclasses import dataclass\n", "from typing import Optional, Tuple, Dict\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Config\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class AlephAttentionConfig:\n", " \"\"\"Configuration for AlephRoutedAttention.\n", "\n", " Args:\n", " dim: model dimension\n", " num_heads: attention heads\n", " mode: 'hub' (linear, GEMM-only) | 'bucket' (hard-address cliques)\n", " K: codebook axes (oriented axes = 2K). Rank/bandwidth knob.\n", " D_addr: address-space dimension (rows live on S^(D_addr-1))\n", " tau: address temperature. Small -> near-discrete routing,\n", " large -> mean-pool collapse. aleph reference: 0.1\n", " codebook_init: 'random' | 'fibonacci' | (K, D_addr) tensor/array\n", " (transplant a trained AlephModel codebook here)\n", " freeze_codebook: register codebook as a buffer (no gradient). Only safe\n", " once a drift check confirms the init IS the attractor.\n", " causal: autoregressive masking (both modes)\n", " chunk_size: hub-causal chunk width (exact chunked linear attention)\n", " block_size: bucket-mode sorted-window width W (keys window = 2W\n", " via 1-block lookback)\n", " bucket_bias_scale_init: init of the learnable scale on the differentiable\n", " address-agreement bias (the codebook's gradient path in\n", " bucket mode)\n", " confidence_gate: multiply head outputs by aleph address confidence\n", " ||(p+ - p-) @ A|| (experimental; default off)\n", " qkv_bias / out_bias: projection biases\n", " dropout: output-projection dropout ONLY (never on the geometric path)\n", " eps: numerical floor for denominators\n", " \"\"\"\n", " dim: int = 512\n", " num_heads: int = 8\n", " mode: str = \"hub\" # 'hub' | 'bucket'\n", " K: int = 64\n", " D_addr: int = 4\n", " tau: float = 0.1\n", " codebook_init: object = \"fibonacci\"\n", " freeze_codebook: bool = False\n", " causal: bool = False\n", " chunk_size: int = 128\n", " block_size: int = 64\n", " bucket_bias_scale_init: float = 1.0\n", " confidence_gate: bool = False\n", " tied_address: bool = False # share q/k address projection. EMPIRICAL (2026-06-09\n", " # CPU recall A/B): tying HURTS — sharp self-affinity\n", " # at low tau structurally biases routing to self\n", " # (same family as softmax(1/d) collapse). Keep False.\n", " qkv_bias: bool = False\n", " out_bias: bool = True\n", " dropout: float = 0.0\n", " eps: float = 1e-8\n", "\n", " def __post_init__(self):\n", " assert self.mode in (\"hub\", \"bucket\"), f\"mode must be 'hub'|'bucket', got {self.mode!r}\"\n", " assert self.dim % self.num_heads == 0, \\\n", " f\"dim ({self.dim}) must be divisible by num_heads ({self.num_heads})\"\n", " self.head_dim = self.dim // self.num_heads\n", " assert self.K >= 2 and self.D_addr >= 2\n", " assert self.tau > 0 and self.eps > 0\n", " assert self.chunk_size > 0 and self.block_size > 0\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Codebook init (ported from geolip-svae aleph_model.py — self-contained)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _super_fibonacci_s3(n: int, dtype=torch.float32) -> Tensor:\n", " \"\"\"n near-uniform unit quaternions on S^3 via super-Fibonacci spirals\n", " (Alexa, CVPR 2022). Deterministic, low-discrepancy. Returns (n, 4).\"\"\"\n", " PHI = math.sqrt(2.0)\n", " PSI = 1.533751168755204288118041\n", " i = torch.arange(n, dtype=torch.float64) + 0.5\n", " s = i / n\n", " r = torch.sqrt(s)\n", " R = torch.sqrt(1.0 - s)\n", " alpha = 2.0 * math.pi * i / PHI\n", " beta = 2.0 * math.pi * i / PSI\n", " q = torch.stack([r * torch.sin(alpha), r * torch.cos(alpha),\n", " R * torch.sin(beta), R * torch.cos(beta)], dim=-1)\n", " return q.to(dtype)\n", "\n", "\n", "def _init_codebook(K: int, D: int, init, dtype=torch.float32) -> Tensor:\n", " \"\"\"'random' Gaussian | 'fibonacci' near-uniform spread (exact at D=4,\n", " seeded-normalized fallback otherwise) | caller (K, D) array, row-normalized\n", " — the transplant path for a trained AlephModel codebook.\"\"\"\n", " if isinstance(init, str):\n", " if init == \"random\":\n", " return torch.randn(K, D, dtype=dtype)\n", " if init == \"fibonacci\":\n", " if D == 4:\n", " return F.normalize(_super_fibonacci_s3(K, dtype=dtype), dim=-1)\n", " g = torch.Generator().manual_seed(0)\n", " return F.normalize(torch.randn(K, D, generator=g, dtype=dtype), dim=-1)\n", " raise ValueError(f\"unknown codebook_init '{init}'\")\n", " A = torch.as_tensor(init, dtype=dtype)\n", " if tuple(A.shape) != (K, D):\n", " raise ValueError(f\"codebook_init array shape {tuple(A.shape)} != ({K}, {D})\")\n", " return F.normalize(A, dim=-1)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Main module\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class AlephRoutedAttention(nn.Module):\n", " \"\"\"Attention routed through a learned (K, D_addr) projective codebook.\n", "\n", " Per head, queries and keys are projected to D_addr and sphere-normalized.\n", " The aleph address p(x) = softmax([u; -u]), u = (x_hat @ A^T) / tau\n", " is the routing medium:\n", "\n", " mode='hub' tokens communicate THROUGH the codebook — linear attention\n", " with p as the feature map, antipodal-factored to K-wide ops.\n", " mode='bucket' tokens attend only to same-winner-half-axis peers — exact\n", " softmax within sorted blocks.\n", "\n", " forward(x, attn_mask=None) -> Tensor (B, S, dim). Diagnostics: address_stats().\n", " \"\"\"\n", "\n", " def __init__(self, config: AlephAttentionConfig):\n", " super().__init__()\n", " self.cfg = config\n", " c = config\n", " self.dim, self.H, self.hd = c.dim, c.num_heads, c.head_dim\n", " self.K, self.Da, self.tau = c.K, c.D_addr, c.tau\n", "\n", " # ── projections ──\n", " # address projections: per-head D_addr rows for q and k (the routing space)\n", " self.q_addr = nn.Linear(c.dim, self.H * self.Da, bias=c.qkv_bias)\n", " nn.init.orthogonal_(self.q_addr.weight) # load-bearing convention\n", " if c.tied_address:\n", " self.k_addr = self.q_addr # one routing space\n", " else:\n", " self.k_addr = nn.Linear(c.dim, self.H * self.Da, bias=c.qkv_bias)\n", " nn.init.orthogonal_(self.k_addr.weight)\n", " # value projection: full head_dim payload\n", " self.v_proj = nn.Linear(c.dim, c.dim, bias=c.qkv_bias)\n", " self.out_proj = nn.Linear(c.dim, c.dim, bias=c.out_bias)\n", " self.dropout = nn.Dropout(c.dropout) # output path only\n", "\n", " # bucket mode additionally scores with full-width q/k (payload attention\n", " # inside the clique); hub routes purely through the address\n", " if c.mode == \"bucket\":\n", " self.q_proj = nn.Linear(c.dim, c.dim, bias=c.qkv_bias)\n", " self.k_proj = nn.Linear(c.dim, c.dim, bias=c.qkv_bias)\n", " nn.init.orthogonal_(self.q_proj.weight)\n", " nn.init.orthogonal_(self.k_proj.weight)\n", " self.bucket_bias_scale = nn.Parameter(\n", " torch.tensor(float(c.bucket_bias_scale_init)))\n", " self.scale = 1.0 / math.sqrt(self.hd)\n", "\n", " # ── the aleph codebook ──\n", " A0 = _init_codebook(c.K, c.D_addr, c.codebook_init)\n", " if c.freeze_codebook:\n", " self.register_buffer(\"codebook\", A0)\n", " else:\n", " self.codebook = nn.Parameter(A0)\n", "\n", " # diversity-loss hook: stash the mean address (WITH grad) when armed\n", " self.emit_diversity: bool = False\n", " self._mean_address: Optional[Tensor] = None # (2K,) when armed\n", "\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # Address machinery (shared)\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", " def oriented_codebook(self) -> Tensor:\n", " \"\"\"(2K, D_addr) oriented half-axes [+A; -A], unit rows.\"\"\"\n", " A = F.normalize(self.codebook, dim=-1)\n", " return torch.cat([A, -A], dim=0)\n", "\n", " def export_codebook(self) -> Tensor:\n", " \"\"\"Normalized (K, D_addr) axes for the geolip-svae antipodal-collapse\n", " extraction — the preregistered basin test entry point.\"\"\"\n", " return F.normalize(self.codebook.detach(), dim=-1).cpu()\n", "\n", " def _split_addr(self, t: Tensor, B: int, S: int) -> Tensor:\n", " \"\"\"(B, S, H*Da) -> (B, H, S, Da), rows sphere-normalized.\"\"\"\n", " t = t.view(B, S, self.H, self.Da).transpose(1, 2)\n", " return F.normalize(t, dim=-1) # S^(D_addr-1): the premise\n", "\n", " def _address(self, x_hat: Tensor) -> Tuple[Tensor, Tensor]:\n", " \"\"\"Aleph address of unit rows x_hat (..., Da) against the codebook.\n", "\n", " Returns (p_plus, p_minus), each (..., K), with\n", " p_plus_k = e^{ u_k} / Z, p_minus_k = e^{-u_k} / Z,\n", " Z = sum_k (e^{u_k} + e^{-u_k}), u = (x_hat @ A^T)/tau\n", " i.e. the exact softmax over the 2K oriented axes, antipodally factored:\n", " the 2K tensor is never materialized. Stable via max|u| subtraction\n", " (at least one exponent is exactly e^0, so Z' >= 1).\"\"\"\n", " A = F.normalize(self.codebook, dim=-1) # (K, Da)\n", " u = (x_hat @ A.t()) * (1.0 / self.tau) # (..., K) signed\n", " m = u.abs().amax(dim=-1, keepdim=True)\n", " ep = torch.exp(u - m) # ∝ e^{+u}\n", " en = torch.exp(-u - m) # ∝ e^{-u}\n", " Z = (ep + en).sum(dim=-1, keepdim=True) # >= 1 by construction\n", " return ep / Z, en / Z\n", "\n", " def _confidence(self, pq_p: Tensor, pq_m: Tensor) -> Tensor:\n", " \"\"\"Aleph address confidence ||(p+ - p-) @ A|| in (0, 1] — the norm of the\n", " soft codebook reconstruction (the hub analogue of ||M_hat||).\"\"\"\n", " A = F.normalize(self.codebook, dim=-1)\n", " return ((pq_p - pq_m) @ A).norm(dim=-1) # (..., )\n", "\n", " def _stash_diversity(self, pk_p: Tensor, pk_m: Tensor,\n", " mask: Optional[Tensor]) -> None:\n", " \"\"\"Mean address over valid key rows -> (2K,) with grad, for diversity_loss().\"\"\"\n", " if not (self.emit_diversity and self.training):\n", " return\n", " if mask is not None:\n", " w = mask[:, None, :, None].to(pk_p.dtype) # (B,1,S,1)\n", " n = w.sum().clamp_min(1.0) * self.H\n", " mp = (pk_p * w).sum(dim=(0, 1, 2)) / n\n", " mm = (pk_m * w).sum(dim=(0, 1, 2)) / n\n", " else:\n", " mp = pk_p.mean(dim=(0, 1, 2))\n", " mm = pk_m.mean(dim=(0, 1, 2))\n", " self._mean_address = torch.cat([mp, mm], dim=0) # (2K,)\n", "\n", " def diversity_loss(self) -> Tensor:\n", " \"\"\"Anti-collapse term (train_aleph div_weight semantics):\n", " log(2K) - H(mean address). Zero at uniform usage. Arm with\n", " model.emit_diversity = True; read after forward; weight ~0.01.\"\"\"\n", " if self._mean_address is None:\n", " return torch.zeros((), device=self.codebook.device)\n", " p = self._mean_address.clamp_min(1e-12)\n", " H = -(p * p.log()).sum()\n", " return math.log(2 * self.K) - H\n", "\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # HUB mode — linear attention through the codebook (pure GEMM)\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", " def _hub_full(self, pq_p, pq_m, pk_p, pk_m, v) -> Tensor:\n", " \"\"\"Non-causal hub. p*: (B,H,S,K), v: (B,H,S,hd) -> (B,H,S,hd).\n", "\n", " score(i,j) = pq+(i)·pk+(j) + pq-(i)·pk-(j) factors through two K-wide\n", " memories; out_i = num_i / den_i with den strictly positive.\"\"\"\n", " Mp = torch.einsum('bhsk,bhsd->bhkd', pk_p, v) # (B,H,K,hd)\n", " Mm = torch.einsum('bhsk,bhsd->bhkd', pk_m, v)\n", " zp = pk_p.sum(dim=2) # (B,H,K)\n", " zm = pk_m.sum(dim=2)\n", " num = torch.einsum('bhsk,bhkd->bhsd', pq_p, Mp) \\\n", " + torch.einsum('bhsk,bhkd->bhsd', pq_m, Mm)\n", " den = torch.einsum('bhsk,bhk->bhs', pq_p, zp) \\\n", " + torch.einsum('bhsk,bhk->bhs', pq_m, zm)\n", " return num / den.unsqueeze(-1).clamp_min(self.cfg.eps)\n", "\n", " def _hub_causal(self, pq_p, pq_m, pk_p, pk_m, v,\n", " state: Optional[Tuple[Tensor, ...]] = None\n", " ) -> Tuple[Tensor, Tuple[Tensor, ...]]:\n", " \"\"\"Exact chunked causal hub: running K-wide state across chunks +\n", " lower-triangular intra-chunk correction. Loop count = S/chunk_size\n", " (the standard chunked linear-attention recurrence — not a per-token loop).\n", "\n", " `state` = (Mp, Mm, zp, zm) carried from previous segments. The state is\n", " constant-size — (B,H,K,hd)+(B,H,K) per sign — regardless of how much\n", " past it summarizes: Mp/Mm are what has been written to each oriented\n", " codebook axis so far. Returns (out, final_state) for streaming.\"\"\"\n", " B, H, S, _ = v.shape\n", " C = min(self.cfg.chunk_size, S)\n", " if state is None:\n", " Mp = v.new_zeros(B, H, self.K, self.hd)\n", " Mm = v.new_zeros(B, H, self.K, self.hd)\n", " zp = v.new_zeros(B, H, self.K)\n", " zm = v.new_zeros(B, H, self.K)\n", " else:\n", " Mp, Mm, zp, zm = state\n", " outs = []\n", " tri_cache: Dict[int, Tensor] = {}\n", " for s0 in range(0, S, C):\n", " s1 = min(s0 + C, S)\n", " qp, qm = pq_p[:, :, s0:s1], pq_m[:, :, s0:s1]\n", " kp, km = pk_p[:, :, s0:s1], pk_m[:, :, s0:s1]\n", " vc = v[:, :, s0:s1]\n", " c = s1 - s0\n", " if c not in tri_cache:\n", " tri_cache[c] = torch.tril(\n", " torch.ones(c, c, device=v.device, dtype=v.dtype))\n", " tri = tri_cache[c]\n", " # intra-chunk (causal) scores — strictly positive entries pre-mask\n", " intra = (torch.einsum('bhik,bhjk->bhij', qp, kp)\n", " + torch.einsum('bhik,bhjk->bhij', qm, km)) * tri\n", " num = intra @ vc \\\n", " + torch.einsum('bhsk,bhkd->bhsd', qp, Mp) \\\n", " + torch.einsum('bhsk,bhkd->bhsd', qm, Mm)\n", " den = intra.sum(dim=-1) \\\n", " + torch.einsum('bhsk,bhk->bhs', qp, zp) \\\n", " + torch.einsum('bhsk,bhk->bhs', qm, zm)\n", " outs.append(num / den.unsqueeze(-1).clamp_min(self.cfg.eps))\n", " # state update (inclusive of this chunk, for the next one)\n", " Mp = Mp + torch.einsum('bhck,bhcd->bhkd', kp, vc)\n", " Mm = Mm + torch.einsum('bhck,bhcd->bhkd', km, vc)\n", " zp = zp + kp.sum(dim=2)\n", " zm = zm + km.sum(dim=2)\n", " return torch.cat(outs, dim=2), (Mp, Mm, zp, zm)\n", "\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # BUCKET mode — hard-address cliques (sort + windowed exact attention)\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", " @staticmethod\n", " def _take(t: Tensor, idx: Tensor) -> Tensor:\n", " \"\"\"Gather along dim=2. t: (B,H,S,X) or (B,H,S); idx: (B,H,S).\"\"\"\n", " if t.dim() == 3:\n", " return torch.gather(t, 2, idx)\n", " return torch.gather(t, 2, idx.unsqueeze(-1).expand(-1, -1, -1, t.shape[-1]))\n", "\n", " @staticmethod\n", " def _window(t: Tensor, nb: int, W: int) -> Tensor:\n", " \"\"\"Blocked tensor (B,H,nb,W,...) -> (B,H,nb,2W,...) keys window =\n", " [previous block ; this block]. Block 0's previous half is junk —\n", " callers must kill it via the validity window.\"\"\"\n", " prev = torch.cat([torch.zeros_like(t[:, :, :1]), t[:, :, :-1]], dim=2)\n", " return torch.cat([prev, t], dim=3)\n", "\n", " def _bucket_attend(self, q, k, v, pq_p, pq_m, pk_p, pk_m,\n", " mask: Optional[Tensor]) -> Tensor:\n", " \"\"\"q,k,v: (B,H,S,hd); p*: (B,H,S,K); mask: (B,S) 1=valid or None.\n", "\n", " 1. bucket = winner oriented half-axis (argmax |u|, sign-resolved)\n", " 2. stable-sort tokens by bucket; pad S to a multiple of W\n", " 3. exact softmax attention within [prev block ; block] windows,\n", " masked to same-bucket, valid, (and causal by original position)\n", " 4. differentiable codebook path: scores += scale * address-agreement\n", " 5. inverse-permute, un-pad.\"\"\"\n", " cfg = self.cfg\n", " B, H, S, hd = q.shape\n", " W = min(cfg.block_size, max(8, S))\n", " dev = q.device\n", "\n", " # ── 1. hard bucket ids ── (recover signed u from the address: u = (log ep - log en)/2\n", " # is unnecessary — argmax of p_plus vs p_minus IS argmax |u| with sign)\n", " win_p, idx_p = pq_p.max(dim=-1) # query side unused for ids\n", " # bucket from the KEY/QUERY shared address rows: use each token's own address\n", " # (q-side and k-side addresses may differ; routing identity = q-address for\n", " # queries, k-address for keys — a token can listen in one clique and speak in\n", " # another. We bucket by the K-side address for keys and Q-side for queries,\n", " # then require equality — implemented by bucketing each side independently.)\n", " def hard_ids(pp: Tensor, pm: Tensor) -> Tensor:\n", " vp, ip = pp.max(dim=-1)\n", " vm, im = pm.max(dim=-1)\n", " plus_wins = vp >= vm\n", " return torch.where(plus_wins, ip, im + self.K) # (B,H,S) in [0, 2K)\n", "\n", " bq = hard_ids(pq_p, pq_m)\n", " bk = hard_ids(pk_p, pk_m)\n", " valid = (mask if mask is not None\n", " else torch.ones(B, S, device=dev, dtype=torch.bool))\n", " valid = valid.bool()[:, None, :].expand(B, H, S)\n", " JUNK = 2 * self.K + 1\n", " bq = torch.where(valid, bq, torch.full_like(bq, JUNK))\n", " bk = torch.where(valid, bk, torch.full_like(bk, JUNK))\n", "\n", " # ── 2. pad to multiple of W, sort by key-bucket ──\n", " pad = (-S) % W\n", " if pad:\n", " def padS(t, fill=0.0):\n", " shape = list(t.shape); shape[2] = pad\n", " return torch.cat([t, t.new_full(shape, fill)], dim=2)\n", " q, k, v = padS(q), padS(k), padS(v)\n", " pq_p, pq_m, pk_p, pk_m = padS(pq_p), padS(pq_m), padS(pk_p), padS(pk_m)\n", " bq, bk = padS(bq, JUNK), padS(bk, JUNK)\n", " valid = padS(valid, False)\n", " Sp = S + pad\n", " nb = Sp // W\n", " pos = torch.arange(Sp, device=dev).view(1, 1, Sp).expand(B, H, Sp)\n", "\n", " sort_idx = bk.argsort(dim=-1, stable=True) # cluster keys by bucket\n", " inv_idx = sort_idx.argsort(dim=-1)\n", " gq, gk, gv = self._take(q, sort_idx), self._take(k, sort_idx), self._take(v, sort_idx)\n", " gpq_p, gpq_m = self._take(pq_p, sort_idx), self._take(pq_m, sort_idx)\n", " gpk_p, gpk_m = self._take(pk_p, sort_idx), self._take(pk_m, sort_idx)\n", " gbq, gbk = self._take(bq, sort_idx), self._take(bk, sort_idx)\n", " gvalid, gpos = self._take(valid.long(), sort_idx).bool(), self._take(pos, sort_idx)\n", "\n", " def blk(t):\n", " return t.view(B, H, nb, W, *t.shape[3:])\n", " q_b, v_b = blk(gq), blk(gv)\n", " k_w = self._window(blk(gk), nb, W) # (B,H,nb,2W,hd)\n", " v_w = self._window(blk(gv), nb, W)\n", " pkp_w = self._window(blk(gpk_p), nb, W)\n", " pkm_w = self._window(blk(gpk_m), nb, W)\n", " bq_b = blk(gbq)\n", " bk_w = self._window(blk(gbk).unsqueeze(-1), nb, W).squeeze(-1)\n", " val_w = self._window(blk(gvalid.long()).unsqueeze(-1), nb, W).squeeze(-1).bool()\n", " pos_b = blk(gpos)\n", " pos_w = self._window(blk(gpos).unsqueeze(-1), nb, W).squeeze(-1)\n", " val_w[:, :, 0, :W] = False # block 0 has no previous\n", "\n", " # ── 3. scores: payload q·k within the window ──\n", " scores = torch.einsum('bhnwd,bhnud->bhnwu', q_b, k_w) * self.scale\n", "\n", " # ── 4. differentiable address-agreement bias (codebook gradient path) ──\n", " pqp_b, pqm_b = blk(gpq_p), blk(gpq_m)\n", " agreement = torch.einsum('bhnwk,bhnuk->bhnwu', pqp_b, pkp_w) \\\n", " + torch.einsum('bhnwk,bhnuk->bhnwu', pqm_b, pkm_w)\n", " scores = scores + self.bucket_bias_scale * agreement\n", "\n", " # ── masks: same bucket, valid, causal ──\n", " same = bq_b.unsqueeze(-1) == bk_w.unsqueeze(-2) # (B,H,nb,W,2W)\n", " keep = same & val_w.unsqueeze(-2)\n", " if cfg.causal:\n", " keep = keep & (pos_w.unsqueeze(-2) <= pos_b.unsqueeze(-1))\n", " scores = scores.masked_fill(~keep, float('-inf'))\n", " attn = F.softmax(scores, dim=-1)\n", " attn = torch.nan_to_num(attn, nan=0.0) # all-masked rows = pads only\n", " out_b = torch.einsum('bhnwu,bhnud->bhnwd', attn, v_w)\n", "\n", " # ── 5. inverse permute, un-pad ──\n", " out = out_b.reshape(B, H, Sp, hd)\n", " out = self._take(out, inv_idx)\n", " return out[:, :, :S]\n", "\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # Forward (returns a single Tensor — compile-rule compliant)\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", " def forward(self, x: Tensor, attn_mask: Optional[Tensor] = None) -> Tensor:\n", " \"\"\"x: (B, S, dim); attn_mask: (B, S) with 1 = valid, 0 = padding.\n", " Returns (B, S, dim).\"\"\"\n", " B, S, _ = x.shape\n", " cfg = self.cfg\n", "\n", " qh = self._split_addr(self.q_addr(x), B, S) # (B,H,S,Da) on the sphere\n", " kh = self._split_addr(self.k_addr(x), B, S)\n", " v = self.v_proj(x).view(B, S, self.H, self.hd).transpose(1, 2)\n", "\n", " pq_p, pq_m = self._address(qh) # (B,H,S,K) each\n", " pk_p, pk_m = self._address(kh)\n", " self._stash_diversity(pk_p, pk_m, attn_mask)\n", "\n", " if attn_mask is not None:\n", " mk = attn_mask[:, None, :, None].to(v.dtype) # kill masked KEYS\n", " pk_p, pk_m, v_in = pk_p * mk, pk_m * mk, v * mk\n", " else:\n", " v_in = v\n", "\n", " if cfg.mode == \"hub\":\n", " if cfg.causal:\n", " out, _ = self._hub_causal(pq_p, pq_m, pk_p, pk_m, v_in)\n", " else:\n", " out = self._hub_full(pq_p, pq_m, pk_p, pk_m, v_in)\n", " else: # bucket\n", " qf = self.q_proj(x).view(B, S, self.H, self.hd).transpose(1, 2)\n", " kf = self.k_proj(x).view(B, S, self.H, self.hd).transpose(1, 2)\n", " out = self._bucket_attend(qf, kf, v, pq_p, pq_m, pk_p, pk_m, attn_mask)\n", "\n", " if cfg.confidence_gate:\n", " out = out * self._confidence(pq_p, pq_m).unsqueeze(-1)\n", "\n", " out = out.transpose(1, 2).reshape(B, S, self.dim)\n", " return self.dropout(self.out_proj(out))\n", "\n", " def forward_stream(self, x: Tensor,\n", " state: Optional[Tuple[Tensor, ...]] = None,\n", " attn_mask: Optional[Tensor] = None\n", " ) -> Tuple[Tensor, Tuple[Tensor, ...]]:\n", " \"\"\"Segment-recurrent forward (mode='hub', causal=True only).\n", "\n", " Processes a segment with the codebook memory carried in `state`\n", " (init None = empty past), returns (out, new_state). Context is\n", " unbounded at constant memory: state is (Mp, Mm, zp, zm), shape\n", " (B,H,K,hd)x2 + (B,H,K)x2, independent of total past length.\n", " TBPTT discipline: .detach() each state tensor between backward\n", " passes — graphs are freed per segment.\"\"\"\n", " assert self.cfg.mode == \"hub\" and self.cfg.causal, \\\n", " \"forward_stream requires mode='hub', causal=True (bucket sorts globally)\"\n", " B, S, _ = x.shape\n", " qh = self._split_addr(self.q_addr(x), B, S)\n", " kh = self._split_addr(self.k_addr(x), B, S)\n", " v = self.v_proj(x).view(B, S, self.H, self.hd).transpose(1, 2)\n", " pq_p, pq_m = self._address(qh)\n", " pk_p, pk_m = self._address(kh)\n", " self._stash_diversity(pk_p, pk_m, attn_mask)\n", " if attn_mask is not None:\n", " mk = attn_mask[:, None, :, None].to(v.dtype)\n", " pk_p, pk_m, v = pk_p * mk, pk_m * mk, v * mk\n", " out, new_state = self._hub_causal(pq_p, pq_m, pk_p, pk_m, v, state)\n", " if self.cfg.confidence_gate:\n", " out = out * self._confidence(pq_p, pq_m).unsqueeze(-1)\n", " out = out.transpose(1, 2).reshape(B, S, self.dim)\n", " return self.dropout(self.out_proj(out)), new_state\n", "\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # Diagnostics (eval-only; never in the hot path)\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", " @torch.no_grad()\n", " def address_stats(self, x: Tensor,\n", " attn_mask: Optional[Tensor] = None,\n", " max_rows: Optional[int] = None) -> Dict[str, float]:\n", " \"\"\"Codebook-health monitors (train_aleph semantics):\n", " perplexity : exp(H(mean address)) — effective oriented axes in use,\n", " in [1, 2K]. The collapse detector.\n", " margin : mean (top1 - top2) of per-row address — decisiveness.\n", " confidence : mean ||(p+ - p-) @ A|| — address sharpness in (0, 1].\n", " bucket_cv : coefficient of variation of hard-bucket occupancy\n", " (load-balance; bucket mode's health metric).\n", " \"\"\"\n", " B, S, _ = x.shape\n", " kh = self._split_addr(self.k_addr(x), B, S)\n", " pp, pm = self._address(kh)\n", " if attn_mask is not None:\n", " m = attn_mask.bool()[:, None, :].expand(B, self.H, S)\n", " pp = pp[m]; pm = pm[m] # (R, K)\n", " else:\n", " pp = pp.reshape(-1, self.K); pm = pm.reshape(-1, self.K)\n", " full = torch.cat([pp, pm], dim=-1) # (R, 2K)\n", " if max_rows is not None and full.shape[0] > max_rows:\n", " full = full[torch.randperm(full.shape[0])[:max_rows]]\n", " pp, pm = full[:, :self.K], full[:, self.K:]\n", "\n", " mean_addr = full.mean(dim=0).clamp_min(1e-12)\n", " H = -(mean_addr * mean_addr.log()).sum()\n", " perplexity = H.exp().item()\n", "\n", " top2 = full.topk(2, dim=-1).values\n", " margin = (top2[:, 0] - top2[:, 1]).mean().item()\n", "\n", " A = F.normalize(self.codebook, dim=-1)\n", " confidence = ((pp - pm) @ A).norm(dim=-1).mean().item()\n", "\n", " ids = full.argmax(dim=-1)\n", " occ = torch.bincount(ids, minlength=2 * self.K).float()\n", " bucket_cv = (occ.std(unbiased=False) / occ.mean().clamp_min(1e-12)).item()\n", "\n", " return {\"perplexity\": perplexity, \"margin\": margin,\n", " \"confidence\": confidence, \"bucket_cv\": bucket_cv,\n", " \"max_perplexity\": float(2 * self.K)}\n", "\n", " def extra_repr(self) -> str:\n", " c = self.cfg\n", " return (f\"dim={c.dim}, heads={c.num_heads}, mode={c.mode}, \"\n", " f\"K={c.K} (2K={2*c.K} oriented), D_addr={c.D_addr}, \"\n", " f\"tau={c.tau}, causal={c.causal}\")\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Reference baseline (for the harness A/B)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class StandardAttention(nn.Module):\n", " \"\"\"Plain softmax attention, same I/O contract, for the A/B.\"\"\"\n", "\n", " def __init__(self, dim: int, num_heads: int, causal: bool = False):\n", " super().__init__()\n", " assert dim % num_heads == 0\n", " self.H, self.hd, self.causal = num_heads, dim // num_heads, causal\n", " self.qkv = nn.Linear(dim, 3 * dim, bias=False)\n", " self.out_proj = nn.Linear(dim, dim)\n", " self.scale = 1.0 / math.sqrt(self.hd)\n", "\n", " def forward(self, x: Tensor, attn_mask: Optional[Tensor] = None) -> Tensor:\n", " B, S, D = x.shape\n", " q, k, v = self.qkv(x).view(B, S, 3, self.H, self.hd) \\\n", " .permute(2, 0, 3, 1, 4).unbind(0)\n", " scores = (q @ k.transpose(-2, -1)) * self.scale\n", " if attn_mask is not None:\n", " scores = scores.masked_fill(\n", " ~attn_mask.bool()[:, None, None, :], float('-inf'))\n", " if self.causal:\n", " tri = torch.ones(S, S, device=x.device, dtype=torch.bool).tril()\n", " scores = scores.masked_fill(~tri, float('-inf'))\n", " out = torch.nan_to_num(F.softmax(scores, dim=-1), nan=0.0) @ v\n", " return self.out_proj(out.transpose(1, 2).reshape(B, S, D))\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Harness — associative recall (routing-sensitive synthetic task)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "#\n", "# Sequence = [k1 v1 k2 v2 ... kn vn Q kq] -> predict the value paired with kq.\n", "# Solvable only by routing the query token to the matching key token: a task\n", "# where the routing medium IS the bottleneck. Trained with pure Adam (never\n", "# AdamW — weight decay fights the geometric basin).\n", "\n", "class TinyRecallModel(nn.Module):\n", " def __init__(self, vocab: int, dim: int, attn: nn.Module, n_layers: int = 2,\n", " attn_factory=None):\n", " super().__init__()\n", " self.emb = nn.Embedding(vocab, dim)\n", " self.pos = nn.Parameter(0.02 * torch.randn(1, 512, dim))\n", " layers = []\n", " for i in range(n_layers):\n", " a = attn if (i == 0 and attn_factory is None) else attn_factory()\n", " layers.append(nn.ModuleDict({\n", " \"norm1\": nn.LayerNorm(dim), \"attn\": a,\n", " \"norm2\": nn.LayerNorm(dim),\n", " \"mlp\": nn.Sequential(nn.Linear(dim, 2 * dim), nn.GELU(),\n", " nn.Linear(2 * dim, dim)),\n", " }))\n", " self.layers = nn.ModuleList(layers)\n", " self.head = nn.Linear(dim, vocab)\n", "\n", " def forward(self, ids: Tensor) -> Tensor:\n", " x = self.emb(ids) + self.pos[:, :ids.shape[1]]\n", " for L in self.layers:\n", " x = x + L[\"attn\"](L[\"norm1\"](x))\n", " x = x + L[\"mlp\"](L[\"norm2\"](x))\n", " return self.head(x[:, -1]) # predict from final token\n", "\n", "\n", "def make_recall_batch(B: int, n_pairs: int, n_keys: int, n_vals: int,\n", " device) -> Tuple[Tensor, Tensor]:\n", " \"\"\"Tokens: [0, n_keys) keys | [n_keys, n_keys+n_vals) values | Q = last id.\"\"\"\n", " Q = n_keys + n_vals\n", " keys = torch.stack([torch.randperm(n_keys, device=device)[:n_pairs]\n", " for _ in range(B)]) # unique keys per row\n", " vals = torch.randint(0, n_vals, (B, n_pairs), device=device) + n_keys\n", " seq = torch.stack([keys, vals], dim=-1).reshape(B, 2 * n_pairs)\n", " qi = torch.randint(0, n_pairs, (B,), device=device)\n", " kq = keys.gather(1, qi[:, None])\n", " target = vals.gather(1, qi[:, None]).squeeze(1)\n", " ids = torch.cat([seq, torch.full((B, 1), Q, device=device), kq], dim=1)\n", " return ids, target\n", "\n", "\n", "def run_harness(mode: str, steps: int = 300, device: str = \"cpu\",\n", " seed: int = 1234, log_every: int = 50,\n", " dim: int = 128, n_heads: int = 4, K: int = 32, D_addr: int = 4,\n", " n_pairs: int = 12, n_keys: int = 48, n_vals: int = 24,\n", " batch: int = 64, lr: float = 3e-4,\n", " div_weight: float = 0.0, tied_address: bool = False,\n", " codebook_init=\"fibonacci\", lr_decay: bool = True,\n", " snapshot_codebook: bool = False) -> Dict[str, float]:\n", " torch.manual_seed(seed)\n", " vocab = n_keys + n_vals + 1\n", " if mode == \"standard\":\n", " attn_factory = lambda: StandardAttention(dim, n_heads)\n", " first = attn_factory()\n", " else:\n", " cfg = AlephAttentionConfig(dim=dim, num_heads=n_heads, mode=mode,\n", " K=K, D_addr=D_addr, tau=0.1,\n", " tied_address=tied_address,\n", " codebook_init=codebook_init)\n", " attn_factory = lambda: AlephRoutedAttention(cfg)\n", " first = attn_factory()\n", " model = TinyRecallModel(vocab, dim, first, n_layers=2,\n", " attn_factory=attn_factory).to(device)\n", " opt = torch.optim.Adam(model.parameters(), lr=lr) # pure Adam, never AdamW\n", " sched = (torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=steps,\n", " eta_min=lr * 0.1) if lr_decay else None)\n", "\n", " aleph_layers = [m for m in model.modules() if isinstance(m, AlephRoutedAttention)]\n", " for a in aleph_layers:\n", " a.emit_diversity = div_weight > 0\n", "\n", " print(f\"\\n=== mode={mode} params={sum(p.numel() for p in model.parameters()):,} ===\")\n", " final = {}\n", " snapshots = [] # (step, (K,D)) trajectory\n", " if snapshot_codebook and aleph_layers:\n", " snapshots.append((0, aleph_layers[0].export_codebook()))\n", " for step in range(1, steps + 1):\n", " ids, target = make_recall_batch(batch, n_pairs, n_keys, n_vals, device)\n", " logits = model(ids)\n", " loss = F.cross_entropy(logits, target)\n", " if div_weight > 0:\n", " loss = loss + div_weight * sum(a.diversity_loss() for a in aleph_layers)\n", " opt.zero_grad(set_to_none=True)\n", " loss.backward()\n", " gnorm = torch.nn.utils.clip_grad_norm_(\n", " model.parameters(), max(loss.item(), 1.0)) # Phil's clip rule\n", " opt.step()\n", " if sched is not None:\n", " sched.step()\n", "\n", " if step % log_every == 0 or step == steps:\n", " with torch.no_grad():\n", " acc = (logits.argmax(-1) == target).float().mean().item()\n", " line = f\" step {step:4d} loss {loss.item():.4f} acc {acc:.3f} |g| {gnorm:.2f}\"\n", " if aleph_layers:\n", " model.eval()\n", " x_probe = model.emb(ids) + model.pos[:, :ids.shape[1]]\n", " st = aleph_layers[0].address_stats(x_probe)\n", " model.train()\n", " line += (f\" ppl {st['perplexity']:.1f}/{st['max_perplexity']:.0f}\"\n", " f\" margin {st['margin']:.3f} conf {st['confidence']:.3f}\"\n", " f\" bktCV {st['bucket_cv']:.2f}\")\n", " final.update(st)\n", " print(line)\n", " final.update({\"loss\": loss.item(), \"acc\": acc})\n", " if snapshot_codebook and aleph_layers:\n", " snapshots.append((step, aleph_layers[0].export_codebook()))\n", " if snapshots:\n", " final[\"codebook_snapshots\"] = snapshots\n", " return final\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke tests + activation\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " torch.manual_seed(0)\n", " print(\"=\" * 70)\n", " print(\"AlephRoutedAttention — smoke tests\")\n", " print(\"=\" * 70)\n", "\n", " for mode in (\"hub\", \"bucket\"):\n", " for causal in (False, True):\n", " cfg = AlephAttentionConfig(dim=64, num_heads=4, mode=mode, K=16,\n", " D_addr=4, causal=causal, block_size=16,\n", " chunk_size=32)\n", " m = AlephRoutedAttention(cfg)\n", " x = torch.randn(2, 50, 64, requires_grad=True) # odd S: pad path\n", " mask = torch.ones(2, 50); mask[1, 40:] = 0\n", " y = m(x, attn_mask=mask)\n", " assert y.shape == (2, 50, 64), y.shape\n", " assert torch.isfinite(y).all()\n", " y.sum().backward()\n", " assert torch.isfinite(x.grad).all()\n", " assert m.codebook.grad is not None and torch.isfinite(m.codebook.grad).all(), \\\n", " f\"codebook got no/bad gradient in mode={mode}\"\n", " cb_g = m.codebook.grad.norm().item()\n", " print(f\" ✓ mode={mode:6s} causal={causal!s:5s} out {tuple(y.shape)} \"\n", " f\"codebook |grad|={cb_g:.4f}\")\n", " x.grad = None\n", "\n", " # hub causal == hub full restricted? sanity: causal output at position i must\n", " # not depend on tokens > i. Perturb a late token; early outputs must not move.\n", " cfg = AlephAttentionConfig(dim=64, num_heads=4, mode=\"hub\", K=16, D_addr=4,\n", " causal=True, chunk_size=16)\n", " m = AlephRoutedAttention(cfg).eval()\n", " x = torch.randn(1, 40, 64)\n", " y1 = m(x)\n", " x2 = x.clone(); x2[0, 35] += 10.0\n", " y2 = m(x2)\n", " assert torch.allclose(y1[0, :35], y2[0, :35], atol=1e-5), \"causality leak!\"\n", " print(\" ✓ hub causal: no future leakage (perturbation test)\")\n", "\n", " # stats sanity\n", " st = m.address_stats(x)\n", " assert 1.0 <= st[\"perplexity\"] <= st[\"max_perplexity\"] + 1e-3\n", " print(f\" ✓ stats: {st}\")\n", "\n", " # diversity hook\n", " m2 = AlephRoutedAttention(AlephAttentionConfig(dim=64, num_heads=4, K=16))\n", " m2.train(); m2.emit_diversity = True\n", " _ = m2(torch.randn(2, 20, 64))\n", " d = m2.diversity_loss()\n", " assert d.requires_grad and torch.isfinite(d)\n", " print(f\" ✓ diversity_loss = {d.item():.4f} (grad-carrying)\")\n", "\n", " # fibonacci init at D=4 is unit + deterministic\n", " A = _init_codebook(32, 4, \"fibonacci\")\n", " assert torch.allclose(A.norm(dim=-1), torch.ones(32), atol=1e-5)\n", " print(\" ✓ super-Fibonacci codebook init (D=4) unit rows\")\n", " print(\"All smoke tests passed.\\n\")\n", "\n", "\n", "def _running_in_notebook() -> bool:\n", " \"\"\"Pasted Colab/Jupyter cells run with __name__ == '__main__'; this keeps\n", " the demo/CLI below inert there so pasting never auto-fires it.\"\"\"\n", " try:\n", " from IPython import get_ipython\n", " return get_ipython() is not None\n", " except Exception:\n", " return False\n", "\n", "\n", "if __name__ == \"__main__\" and not _running_in_notebook():\n", " import argparse\n", " ap = argparse.ArgumentParser(description=\"Aleph-routed attention — smoke + A/B harness\")\n", " ap.add_argument(\"--steps\", type=int, default=300)\n", " ap.add_argument(\"--device\", default=\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " ap.add_argument(\"--modes\", nargs=\"+\", default=[\"hub\", \"bucket\", \"standard\"])\n", " ap.add_argument(\"--div-weight\", type=float, default=0.0)\n", " ap.add_argument(\"--K\", type=int, default=32)\n", " ap.add_argument(\"--tau\", type=float, default=0.1)\n", " ap.add_argument(\"--smoke-only\", action=\"store_true\")\n", " # parse_known_args: ignore foreign argv (e.g. Jupyter/Colab injects\n", " # `-f /.../kernel-*.json`), so the module runs in notebooks unchanged\n", " args, _unknown = ap.parse_known_args()\n", "\n", " _smoke()\n", " if not args.smoke_only:\n", " results = {}\n", " for mode in args.modes:\n", " results[mode] = run_harness(mode, steps=args.steps, device=args.device,\n", " K=args.K, div_weight=args.div_weight)\n", " print(\"\\n\" + \"=\" * 70)\n", " print(\"A/B summary (associative recall)\")\n", " for mode, r in results.items():\n", " extra = (f\" ppl {r.get('perplexity', float('nan')):.1f}\"\n", " f\" margin {r.get('margin', float('nan')):.3f}\"\n", " if \"perplexity\" in r else \"\")\n", " print(f\" {mode:9s} loss {r['loss']:.4f} acc {r['acc']:.3f}{extra}\")\n", " print(\"=\" * 70)\n", " print(\"\\nBasin test entry point: model.export_codebook() -> feed to the\")\n", " print(\"geolip-svae antipodal-collapse extraction. Preregistered criterion:\")\n", " print(\"|deviation| < 0.05 on RP^(D-1) = cross-objective attractor evidence.\")" ], "metadata": { "id": "SwRSp0PG5Z5h" }, "execution_count": 5, "outputs": [] }, { "cell_type": "code", "source": [ "# aleph_trigram_lm.py\n", "\"\"\"\n", "Aleph-Routed Trigram LM — the basin-test vehicle\n", "=================================================\n", "\n", "Causal language model over the SAME substrate the geolip-svae aleph batteries\n", "were fed: WikiText-103 as a raw UTF-8 byte stream, trigram-packed (3 bytes per\n", "position, stride 3 — the sequence analogue of ByteTrigramDataset's 3-bytes-per-\n", "cell RGB encoding). One token position = one trigram, embedded byte-factored\n", "(sum of 3 byte embeddings + position) and predicted as 3 independent 256-way\n", "byte heads — exactly the substrate, no 256^3 softmax.\n", "\n", "Purpose (preregistered — CORRECTED per RESEARCH_HISTORY.md):\n", " The program established (Phase 2, discoveries #13/#15) at least TWO stable\n", " codebook statutes, SELECTED BY SUBSTRATE: uniform-class (|dev| < 0.05; the\n", " noise solvers) and polytope-class (dev > +0.05, pair fraction >= 45%;\n", " repulsive packing — the BYTE-TRIGRAM solvers, in-distribution dev +0.083).\n", " dev < -0.05 is degenerate (clumping) — the failure statute. Statute is a\n", " property of (model x calibration), not the model alone.\n", "\n", " Therefore, on THIS substrate, the basin question is statute-resolved:\n", " - random init -> STATUTE-SELECTION test. Codebook settling polytope-class\n", " (the substrate-matched statute) or uniform-class under pure attention\n", " gradients = cross-objective attractor evidence. Degenerate = failure.\n", " - fibonacci init -> DRIFT test. Starts in the uniform basin; migration\n", " OUT toward polytope under the symbolic substrate = substrate-driven\n", " statute selection in a new objective (the stronger result).\n", " Run BOTH inits. Statute (deviation + pair fraction, computed per the\n", " program's own Sec 3.11 definitions) is logged per snapshot inline below;\n", " void-richness (beta_2/axis via ripser on projective angular distances,\n", " the symbolic-substrate fingerprint, discovery #20) is the deeper follow-up\n", " on the saved snapshots. Note the two non-interchangeable \"margins\"\n", " (top-1 softmax probability vs projective ||): the inline monitors\n", " detect collapse only; structure evidence is deviation/statute/beta_2.\n", "\n", "Substrate fidelity (mirrors geolip_svae.dataset_presets.ByteTrigramDataset):\n", " - corpus: 'wikitext-103-raw-v1' via the Salesforce/wikitext namespace\n", " (the bare 'wikitext' hub name is deprecated), or any local .txt path\n", " - raw bytes in a single uint8 numpy array (prototypes/CLAUDE.md trap #3:\n", " Python lists of ints are 5-7x the memory; never materialize them)\n", " - seeded window sampling over the stream\n", "Repo invariants honored: pure Adam (never AdamW), no BatchNorm/Dropout on the\n", "geometric path, grad clip = max(task_loss, 1.0).\n", "\n", "Usage (Colab / Blackwell):\n", " try:\n", " from aleph_trigram_lm import TrigramLMConfig, train_trigram_lm\n", " except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", " cfg = TrigramLMConfig(steps=10_000, device='cuda',\n", " attn_mode='hub', codebook_init='random')\n", " result = train_trigram_lm(cfg)\n", " # result['codebook_snapshots'] -> [(step, (K, D_addr) tensor), ...]\n", " # also saved to cfg.snapshot_path for the extraction pass\n", "\n", "Baseline A/B: attn_mode='standard' runs the same LM with softmax attention.\n", "\n", "Author: AbstractPhil + Mirel\n", "Date: 2026-06-09\n", "License: MIT\n", "\"\"\"\n", "\n", "from __future__ import annotations\n", "\n", "import math\n", "import os\n", "import time\n", "from dataclasses import dataclass, field\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import numpy as np\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from aleph_routed_attention import (\n", " AlephRoutedAttention, AlephAttentionConfig, StandardAttention,\n", " )\n", "except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Config\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class TrigramLMConfig:\n", " \"\"\"Everything for one basin run.\n", "\n", " Substrate:\n", " corpus_id: HF dataset config name (default = the aleph batteries'\n", " corpus) OR a local .txt/.text path\n", " max_corpus_bytes: cap on bytes loaded (None = whole corpus, ~520 MB for\n", " wikitext-103). 50–100 MB is plenty for these runs.\n", " seq_len: context length in TRIGRAMS (bytes seen = 3*seq_len)\n", "\n", " Model:\n", " dim/n_layers/n_heads: transformer shell\n", " attn_mode: 'hub' | 'bucket' | 'standard'\n", " K/D_addr/tau: aleph routing knobs (ignored for 'standard')\n", " codebook_init: 'random' for the basin test (MANDATORY there) |\n", " 'fibonacci' | (K, D_addr) array transplant\n", "\n", " Training:\n", " pure Adam + cosine decay to 10%; loss reported in nats and bits/byte.\n", " \"\"\"\n", " # substrate\n", " corpus_id: str = \"wikitext-103-raw-v1\"\n", " split: str = \"train\"\n", " max_corpus_bytes: Optional[int] = 100_000_000\n", " seq_len: int = 256 # trigrams (= 768 bytes of context)\n", " seed: int = 1234\n", "\n", " # model\n", " dim: int = 384\n", " n_layers: int = 4\n", " n_heads: int = 6\n", " attn_mode: str = \"hub\" # 'hub' | 'bucket' | 'standard'\n", " K: int = 64\n", " D_addr: int = 4\n", " tau: float = 0.1\n", " codebook_init: object = \"random\" # basin test requires 'random'\n", " div_weight: float = 0.0 # anti-collapse; run 0 first, observe\n", "\n", " # paradigm + scale\n", " shared_codebook: bool = True # ONE vocabulary, many speakers: all\n", " # layers address the same (K,D) param,\n", " # concentrating address pressure n_layers-x\n", " accum_steps: int = 1 # gradient accumulation (effective batch\n", " # = batch_size * accum_steps)\n", " stream_segments: int = 1 # segments per sample, each seq_len long;\n", " # codebook-memory state carried across\n", " # (TBPTT, detached between segments).\n", " # context = seq_len * stream_segments\n", " # at CONSTANT attention memory. hub-only.\n", " probe_rows: int = 200_000 # address-stats sample size (ppl estimator)\n", "\n", " # training\n", " steps: int = 10_000 # optimizer steps (micro-batches =\n", " # steps * accum_steps)\n", " batch_size: int = 32\n", " lr: float = 3e-4\n", " lr_decay: bool = True\n", " log_every: int = 250\n", " eval_batches: int = 8\n", " device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " amp: bool = False # bf16 autocast on the shell (the\n", " # address stays fp32 inside)\n", "\n", " # outputs\n", " snapshot_codebook: bool = True\n", " snapshot_path: str = \"aleph_lm_codebook_snapshots.pt\"\n", " checkpoint_path: Optional[str] = \"aleph_trigram_lm.pt\"\n", "\n", " def __post_init__(self):\n", " assert self.attn_mode in (\"hub\", \"bucket\", \"standard\")\n", " assert self.dim % self.n_heads == 0\n", " assert self.accum_steps >= 1 and self.stream_segments >= 1\n", " if self.stream_segments > 1:\n", " assert self.attn_mode == \"hub\", \\\n", " \"streaming requires mode='hub' (bucket sorts globally)\"\n", "\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Statute monitor — the program's own diagnostic geometry (Sec 3.11)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _canon(x: Tensor) -> Tensor:\n", " \"\"\"Sign-canonicalize onto RP^(D-1): flip so the first nonzero coord is\n", " positive (antipodes map to one representative).\"\"\"\n", " x = F.normalize(x, dim=-1)\n", " first_nz = x[torch.arange(len(x)), x.abs().argmax(dim=-1)]\n", " return x * torch.sign(first_nz).unsqueeze(-1)\n", "\n", "\n", "def _mean_projective_angle(X: Tensor) -> float:\n", " \"\"\"Mean pairwise acos|cos| over distinct pairs (radians).\"\"\"\n", " c = (X @ X.t()).clamp(-1.0, 1.0).abs()\n", " iu = torch.triu_indices(len(X), len(X), offset=1)\n", " return torch.acos(c[iu[0], iu[1]]).mean().item()\n", "\n", "\n", "_UNIFORM_BASELINE: Dict[int, float] = {}\n", "\n", "def projective_deviation(axes: Tensor, n_ref: int = 4096,\n", " seed: int = 0) -> float:\n", " \"\"\"Uniformity deviation per the program definition: mean pairwise\n", " projective angle of the axes MINUS the same statistic for n_ref uniform\n", " random projective points at the same D. Signed; sign matters.\"\"\"\n", " D = axes.shape[-1]\n", " if D not in _UNIFORM_BASELINE:\n", " g = torch.Generator().manual_seed(seed)\n", " ref = F.normalize(torch.randn(n_ref, D, generator=g), dim=-1)\n", " _UNIFORM_BASELINE[D] = _mean_projective_angle(ref)\n", " return _mean_projective_angle(F.normalize(axes.float(), dim=-1)) \\\n", " - _UNIFORM_BASELINE[D]\n", "\n", "\n", "def antipodal_pair_fraction(axes: Tensor, thresh: float = -0.9) -> float:\n", " \"\"\"Fraction of rows in mutual most-negative pairs with cos < thresh\n", " (the antipodal-collapse acceptance rule).\"\"\"\n", " A = F.normalize(axes.float(), dim=-1)\n", " c = A @ A.t()\n", " c.fill_diagonal_(0.0)\n", " partner = c.argmin(dim=-1)\n", " val = c.gather(-1, partner.unsqueeze(-1)).squeeze(-1)\n", " mutual = partner[partner] == torch.arange(len(A))\n", " return ((val < thresh) & mutual).float().mean().item()\n", "\n", "\n", "def statute(axes: Tensor) -> Dict[str, object]:\n", " \"\"\"Classify per the program taxonomy: dev > +0.05 polytope-class\n", " (repulsive packing); |dev| < 0.05 uniform-class; dev < -0.05 degenerate\n", " (clumping, the failure statute).\"\"\"\n", " dev = projective_deviation(axes)\n", " pf = antipodal_pair_fraction(axes)\n", " cls = (\"polytope\" if dev > 0.05 else\n", " \"degenerate\" if dev < -0.05 else \"uniform\")\n", " return {\"deviation\": dev, \"pair_fraction\": pf, \"statute\": cls}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Substrate — trigram stream (mirrors ByteTrigramDataset's loading)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class TrigramStream:\n", " \"\"\"WikiText-103 (or local .txt) as a uint8 byte stream, sampled as\n", " causal trigram sequences.\n", "\n", " __call__(batch, seq_len) -> (ids, targets):\n", " ids: (B, S, 3) uint8->long — trigram t = bytes[3t : 3t+3]\n", " targets: (B, S, 3) — trigram t+1 (next-trigram prediction)\n", " Windows are sampled at byte offsets aligned to stride 3 so the trigram\n", " framing matches the image packing (cell i = bytes[3i : 3i+3]).\"\"\"\n", "\n", " def __init__(self, corpus_id: str, split: str = \"train\",\n", " max_corpus_bytes: Optional[int] = None, seed: int = 1234):\n", " if os.path.isfile(corpus_id) and corpus_id.endswith((\".txt\", \".text\")):\n", " print(f\"[TrigramStream] loading local corpus {corpus_id} ...\")\n", " with open(corpus_id, \"rb\") as f:\n", " raw = f.read(max_corpus_bytes) if max_corpus_bytes else f.read()\n", " self.stream = np.frombuffer(raw, dtype=np.uint8).copy()\n", " else:\n", " print(f\"[TrigramStream] loading HF corpus {corpus_id} ...\")\n", " from datasets import load_dataset\n", " if corpus_id.startswith(\"wikitext\"):\n", " ds = load_dataset(\"Salesforce/wikitext\", corpus_id, split=split)\n", " else:\n", " ds = load_dataset(corpus_id, split=split)\n", " # accumulate utf-8 bytes directly into a byte buffer — never a\n", " # Python list of ints (prototypes/CLAUDE.md memory trap #3)\n", " buf = bytearray()\n", " cap = max_corpus_bytes or float(\"inf\")\n", " for row in ds:\n", " t = row.get(\"text\", \"\")\n", " if t:\n", " buf.extend(t.encode(\"utf-8\", errors=\"ignore\"))\n", " if len(buf) >= cap:\n", " break\n", " self.stream = np.frombuffer(\n", " bytes(buf[: max_corpus_bytes] if max_corpus_bytes else buf),\n", " dtype=np.uint8).copy()\n", " n_tri = len(self.stream) // 3\n", " print(f\"[TrigramStream] {len(self.stream):,} bytes \"\n", " f\"= {n_tri:,} trigrams\")\n", " assert n_tri > 0, \"corpus too small\"\n", " self._rng = np.random.default_rng(seed)\n", "\n", " def sample(self, batch: int, seq_len: int,\n", " device) -> Tuple[Tensor, Tensor]:\n", " need = 3 * (seq_len + 1) # +1 trigram for targets\n", " hi = len(self.stream) - need\n", " assert hi > 0, f\"corpus shorter than one window ({need} bytes)\"\n", " starts = self._rng.integers(0, hi // 3, size=batch) * 3 # stride-3 aligned\n", " idx = starts[:, None] + np.arange(need)[None, :] # (B, need)\n", " window = self.stream[idx] # (B, need) uint8\n", " tri = torch.from_numpy(window.astype(np.int64)) \\\n", " .view(batch, seq_len + 1, 3)\n", " ids, targets = tri[:, :-1], tri[:, 1:]\n", " return ids.to(device), targets.to(device)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Model — byte-factored trigram LM\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class TrigramLM(nn.Module):\n", " \"\"\"Causal LM over trigram positions. Embedding = sum of three per-slot\n", " byte embeddings (+ learned positions); head = three 256-way byte heads.\n", " Geometric-path hygiene: no BatchNorm/Dropout, pure pre-LN residual shell.\"\"\"\n", "\n", " def __init__(self, cfg: TrigramLMConfig):\n", " super().__init__()\n", " self.cfg = cfg\n", " d = cfg.dim\n", " self.byte_emb = nn.ModuleList([nn.Embedding(256, d) for _ in range(3)])\n", " self.pos = nn.Parameter(0.02 * torch.randn(1, cfg.seq_len, d))\n", "\n", " def make_attn() -> nn.Module:\n", " if cfg.attn_mode == \"standard\":\n", " return StandardAttention(d, cfg.n_heads, causal=True)\n", " return AlephRoutedAttention(AlephAttentionConfig(\n", " dim=d, num_heads=cfg.n_heads, mode=cfg.attn_mode,\n", " K=cfg.K, D_addr=cfg.D_addr, tau=cfg.tau, causal=True,\n", " codebook_init=cfg.codebook_init))\n", "\n", " self.layers = nn.ModuleList([\n", " nn.ModuleDict({\n", " \"norm1\": nn.LayerNorm(d), \"attn\": make_attn(),\n", " \"norm2\": nn.LayerNorm(d),\n", " \"mlp\": nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(),\n", " nn.Linear(4 * d, d)),\n", " }) for _ in range(cfg.n_layers)\n", " ])\n", " self.norm_f = nn.LayerNorm(d)\n", " self.heads = nn.ModuleList([nn.Linear(d, 256) for _ in range(3)])\n", "\n", " # one vocabulary, many speakers: tie every layer's codebook to layer 0's\n", " if cfg.shared_codebook and cfg.attn_mode in (\"hub\", \"bucket\"):\n", " shared = self.layers[0][\"attn\"].codebook\n", " for L in self.layers[1:]:\n", " L[\"attn\"].codebook = shared\n", "\n", " def aleph_layers(self) -> List[AlephRoutedAttention]:\n", " return [m for m in self.modules() if isinstance(m, AlephRoutedAttention)]\n", "\n", " def backbone(self, ids: Tensor) -> Tensor:\n", " \"\"\"ids: (B, S, 3) -> (B, S, dim)\"\"\"\n", " x = sum(emb(ids[..., i]) for i, emb in enumerate(self.byte_emb))\n", " x = x + self.pos[:, : ids.shape[1]]\n", " for L in self.layers:\n", " x = x + L[\"attn\"](L[\"norm1\"](x))\n", " x = x + L[\"mlp\"](L[\"norm2\"](x))\n", " return self.norm_f(x)\n", "\n", " def forward(self, ids: Tensor) -> Tensor:\n", " \"\"\"(B, S, 3) -> byte logits (B, S, 3, 256)\"\"\"\n", " h = self.backbone(ids)\n", " return torch.stack([head(h) for head in self.heads], dim=2)\n", "\n", " def loss(self, ids: Tensor, targets: Tensor) -> Tensor:\n", " \"\"\"Mean cross-entropy per byte (nats/byte). bpb = loss / ln 2.\"\"\"\n", " logits = self(ids) # (B,S,3,256)\n", " return F.cross_entropy(logits.reshape(-1, 256), targets.reshape(-1))\n", "\n", " # ── streaming: segment-recurrent backbone (codebook memory carried) ──\n", " def stream_loss(self, ids: Tensor, targets: Tensor,\n", " states: Optional[List] = None\n", " ) -> Tuple[Tensor, List]:\n", " \"\"\"One SEGMENT with per-layer carried states. states[i] is layer i's\n", " (Mp, Mm, zp, zm) or None. Returns (loss, new_states) — caller detaches\n", " between backward passes (TBPTT-1: grads flow within segment; values\n", " flow forever).\"\"\"\n", " x = sum(emb(ids[..., i]) for i, emb in enumerate(self.byte_emb))\n", " x = x + self.pos[:, : ids.shape[1]]\n", " new_states: List = []\n", " states = states or [None] * len(self.layers)\n", " for L, st in zip(self.layers, states):\n", " a, ns = L[\"attn\"].forward_stream(L[\"norm1\"](x), state=st)\n", " x = x + a\n", " x = x + L[\"mlp\"](L[\"norm2\"](x))\n", " new_states.append(ns)\n", " h = self.norm_f(x)\n", " logits = torch.stack([head(h) for head in self.heads], dim=2)\n", " loss = F.cross_entropy(logits.reshape(-1, 256), targets.reshape(-1))\n", " return loss, new_states\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Training\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def train_trigram_lm(cfg: TrigramLMConfig,\n", " stream: Optional[TrigramStream] = None) -> Dict:\n", " torch.manual_seed(cfg.seed)\n", " dev = torch.device(cfg.device)\n", " stream = stream or TrigramStream(cfg.corpus_id, cfg.split,\n", " cfg.max_corpus_bytes, cfg.seed)\n", " model = TrigramLM(cfg).to(dev)\n", " n_params = sum(p.numel() for p in model.parameters())\n", " opt = torch.optim.Adam(model.parameters(), lr=cfg.lr) # pure Adam, never AdamW\n", " sched = (torch.optim.lr_scheduler.CosineAnnealingLR(\n", " opt, T_max=cfg.steps, eta_min=cfg.lr * 0.1) if cfg.lr_decay else None)\n", "\n", " alephs = model.aleph_layers()\n", " for a in alephs:\n", " a.emit_diversity = cfg.div_weight > 0\n", "\n", " snapshots: List[Tuple[int, Tensor]] = []\n", " if cfg.snapshot_codebook and alephs:\n", " snapshots.append((0, alephs[0].export_codebook()))\n", "\n", " print(f\"\\n=== trigram LM mode={cfg.attn_mode} params={n_params:,} \"\n", " f\"ctx={cfg.seq_len}x{cfg.stream_segments} trigrams \"\n", " f\"({3*cfg.seq_len*cfg.stream_segments} bytes) \"\n", " f\"eff.batch={cfg.batch_size*cfg.accum_steps} \"\n", " f\"shared_cb={cfg.shared_codebook} \"\n", " f\"device={dev} ===\")\n", " autocast = (torch.autocast(device_type=dev.type, dtype=torch.bfloat16)\n", " if cfg.amp and dev.type == \"cuda\" else None)\n", " result: Dict = {\"mode\": cfg.attn_mode, \"params\": n_params}\n", " t0 = time.time()\n", "\n", " segs = cfg.stream_segments\n", " micro_scale = 1.0 / (cfg.accum_steps * segs)\n", " for step in range(1, cfg.steps + 1):\n", " opt.zero_grad(set_to_none=True)\n", " loss_sum, n_micro = 0.0, 0\n", " for _ in range(cfg.accum_steps):\n", " # one long sample, split into `segs` carried segments\n", " ids, targets = stream.sample(cfg.batch_size, cfg.seq_len * segs, dev)\n", " states = None\n", " for s in range(segs):\n", " sl = slice(s * cfg.seq_len, (s + 1) * cfg.seq_len)\n", " seg_ids, seg_tgt = ids[:, sl], targets[:, sl]\n", " if autocast:\n", " with autocast:\n", " if segs > 1:\n", " loss, states = model.stream_loss(seg_ids, seg_tgt, states)\n", " else:\n", " loss = model.loss(seg_ids, seg_tgt)\n", " else:\n", " if segs > 1:\n", " loss, states = model.stream_loss(seg_ids, seg_tgt, states)\n", " else:\n", " loss = model.loss(seg_ids, seg_tgt)\n", " total = loss\n", " if cfg.div_weight > 0 and alephs:\n", " total = total + cfg.div_weight * sum(\n", " a.diversity_loss() for a in alephs)\n", " (total * micro_scale).backward()\n", " loss_sum += loss.item(); n_micro += 1\n", " if states is not None: # TBPTT boundary\n", " states = [tuple(t.detach() for t in st) for st in states]\n", " loss_avg = loss_sum / n_micro\n", " gnorm = torch.nn.utils.clip_grad_norm_(\n", " model.parameters(), max(loss_avg, 1.0)) # the clip rule\n", " opt.step()\n", " if sched is not None:\n", " sched.step()\n", "\n", " if step % cfg.log_every == 0 or step == cfg.steps:\n", " bpb = loss_avg / math.log(2)\n", " rate = (step * cfg.batch_size * cfg.seq_len * segs\n", " * cfg.accum_steps) / (time.time() - t0)\n", " line = (f\" step {step:6d} loss {loss_avg:.4f} \"\n", " f\"bpb {bpb:.3f} |g| {gnorm:.2f} {rate/1e3:.1f}k tri/s\")\n", " if alephs:\n", " model.eval()\n", " with torch.no_grad():\n", " x_probe = model.backbone(\n", " ids[: min(8, cfg.batch_size), -cfg.seq_len:])\n", " st = alephs[0].address_stats(x_probe, max_rows=cfg.probe_rows)\n", " model.train()\n", " line += (f\" ppl {st['perplexity']:.1f}/{st['max_perplexity']:.0f}\"\n", " f\" margin {st['margin']:.4f}\"\n", " f\" conf {st['confidence']:.3f}\")\n", " result.update(st)\n", " if cfg.snapshot_codebook:\n", " snapshots.append((step, alephs[0].export_codebook()))\n", " print(line)\n", " result.update({\"loss\": loss_avg, \"bpb\": bpb, \"step\": step})\n", "\n", " if snapshots:\n", " result[\"codebook_snapshots\"] = snapshots\n", " drift = (snapshots[-1][1] - snapshots[0][1]).norm().item()\n", " result[\"codebook_drift\"] = drift\n", " traj = [(s, statute(cb)) for s, cb in snapshots]\n", " result[\"statute_trajectory\"] = traj\n", " torch.save({\"snapshots\": snapshots, \"statute_trajectory\": traj,\n", " \"config\": cfg.__dict__, \"K\": cfg.K, \"D_addr\": cfg.D_addr},\n", " cfg.snapshot_path)\n", " print(f\"\\n[basin] {len(snapshots)} snapshots -> {cfg.snapshot_path}\"\n", " f\" drift |A_end - A_0| = {drift:.4f}\")\n", " print(\"[basin] statute trajectory (program taxonomy: polytope is the \"\n", " \"substrate-matched\\n statute for byte-trigram; uniform is \"\n", " \"the noise/OOD statute; degenerate = failure):\")\n", " for s, st in traj:\n", " print(f\" step {s:6d} dev {st['deviation']:+.4f} \"\n", " f\"pairs {st['pair_fraction']:.0%} -> {st['statute']}\")\n", " print(\"[basin] deeper follow-up on saved snapshots: beta_2/axis via \"\n", " \"ripser on projective\\n angular distances (the \"\n", " \"void/symbolic fingerprint, discovery #20).\")\n", " if cfg.checkpoint_path:\n", " torch.save({\"model_state_dict\": model.state_dict(),\n", " \"config\": cfg.__dict__}, cfg.checkpoint_path)\n", " print(f\"[ckpt] saved -> {cfg.checkpoint_path}\")\n", " return result\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Activation\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke(device: str = \"cpu\"):\n", " \"\"\"End-to-end on a synthetic local corpus — no downloads.\"\"\"\n", " print(\"=\" * 70)\n", " print(\"aleph_trigram_lm — smoke (synthetic corpus)\")\n", " print(\"=\" * 70)\n", " path = \"/tmp/_smoke_corpus.txt\"\n", " rng = np.random.default_rng(0)\n", " words = [b\"the\", b\"aleph\", b\"address\", b\"routes\", b\"attention\",\n", " b\"through\", b\"a\", b\"learned\", b\"projective\", b\"codebook\"]\n", " with open(path, \"wb\") as f:\n", " f.write(b\" \".join(words[i] for i in rng.integers(0, len(words), 60_000)))\n", " cfg = TrigramLMConfig(corpus_id=path, steps=30, log_every=10,\n", " dim=96, n_layers=2, n_heads=4, K=16, seq_len=64,\n", " batch_size=8, device=device,\n", " snapshot_path=\"/tmp/_smoke_snaps.pt\",\n", " checkpoint_path=None)\n", " r = train_trigram_lm(cfg)\n", " assert \"codebook_snapshots\" in r and len(r[\"codebook_snapshots\"]) >= 2\n", " assert math.isfinite(r[\"loss\"]) and r[\"loss\"] < math.log(256)\n", " print(f\"\\nsmoke OK — loss {r['loss']:.3f} (< ln256={math.log(256):.3f} prior), \"\n", " f\"drift {r['codebook_drift']:.4f}, \"\n", " f\"{len(r['codebook_snapshots'])} snapshots saved\")\n", "\n", "\n", "def _running_in_notebook() -> bool:\n", " \"\"\"Pasted Colab/Jupyter cells run with __name__ == '__main__'; this keeps\n", " the demo/CLI below inert there so pasting never auto-fires it.\"\"\"\n", " try:\n", " from IPython import get_ipython\n", " return get_ipython() is not None\n", " except Exception:\n", " return False\n", "\n", "\n", "if __name__ == \"__main__\" and not _running_in_notebook():\n", " import argparse\n", " ap = argparse.ArgumentParser(description=\"Aleph trigram LM — basin run\")\n", " ap.add_argument(\"--smoke-only\", action=\"store_true\")\n", " ap.add_argument(\"--mode\", default=\"hub\", choices=[\"hub\", \"bucket\", \"standard\"])\n", " ap.add_argument(\"--steps\", type=int, default=10_000)\n", " ap.add_argument(\"--corpus-mb\", type=int, default=100)\n", " ap.add_argument(\"--codebook-init\", default=\"random\")\n", " ap.add_argument(\"--device\",\n", " default=\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " args, _unknown = ap.parse_known_args() # notebook-safe (ignores -f kernel.json)\n", "\n", " if args.smoke_only:\n", " _smoke(device=\"cpu\")\n", " else:\n", " cfg = TrigramLMConfig(attn_mode=args.mode, steps=args.steps,\n", " max_corpus_bytes=args.corpus_mb * 1_000_000,\n", " codebook_init=args.codebook_init,\n", " device=args.device)\n", " train_trigram_lm(cfg)" ], "metadata": { "id": "z5fYejb85cY6" }, "execution_count": 6, "outputs": [] }, { "cell_type": "code", "source": [ "# aleph_lm.py\n", "\"\"\"\n", "AlephLM — prediction through the codebook, with guarantees\n", "===========================================================\n", "\n", "The composite reduction (2026-06-09): a causal trigram LM in which the aleph\n", "signed-projective address is load-bearing at ALL THREE stations — input\n", "addressing, mixing, and prediction. The codebook receives gradient from\n", "routing, from the predicted next-address pi, and from every candidate address\n", "kappa. One geometry, closed loop, smooth everywhere (no argmax in the train\n", "path): differential trigram-to-trigram prediction.\n", "\n", " CODEC bytes -> trigrams g_t (stride 3)\n", " EMBED e_t = sum_c E_c[g_t[c]] (byte-factored)\n", " MIX AlephRoutedAttention hub layers, shared codebook, causal\n", " PREDICT pi = softmax([w; -w]), w = W_pi h_t (free antipodal-tied)\n", " CANDIDATE kappa(tau) = address(normalize(W_k sum_c E_c[tau[c]]))\n", " SCORE logit(tau) = alpha * log( pi+ . kappa+(tau) + pi- . kappa-(tau) )\n", " OUTPUT hybrid: P(g) = g_in * P_bank(g | in) + (1-g_in) * P_byte(g)\n", "\n", "THE GUARANTEE LEDGER (all demonstrated numerically 2026-06-09; see session log):\n", " T1/T2 pi parameterization: address-constrained pi is projectively UNIMODAL\n", " (logits linear in x-hat) — a two-spike target is unreachable (best\n", " joint mass 1.4% vs 50% needed). The free antipodal-tied simplex\n", " represents any tied-logit distribution. DEFAULT: free tied simplex;\n", " address-constrained is the unimodal ablation (pi_mode='address').\n", " T3 Tied [w; -w] implies p+k * p-k is CONSTANT across k — every axis is\n", " forced to an orientation stance. Feature-or-bug: empirical.\n", " T4 The 3x256 byte-product head cannot express within-trigram byte\n", " correlation (rank-1 tensor over 256^3); it is the guaranteed-floor\n", " baseline (head='byte'), not the main head.\n", " T5 The hybrid output is a PROPER full-support distribution and its CE\n", " decomposes exactly: -log P(g) = -log gate_branch - log P_branch(g).\n", " Implemented verbatim. \"Run all three banks\" = ablations inside one\n", " provably-correct machine.\n", " T6 Raw-score softmax over a bank has a sharpness ceiling (scores in\n", " (0,1] => CE floor 7.32 nats at M=4096). Logits are LOG-kernel with a\n", " learnable scale alpha. Non-negotiable.\n", " T7 Output logit rank <= 2K (softmax bottleneck): K governs attention\n", " rank, output rank, and mode capacity — one knob, three proven roles.\n", " T8 The write-head target Delta-z = sum of future addresses is the\n", " ORDER-MARGINALIZED multiset of the next W trigrams (permutation-\n", " invariant by commutativity). It predicts WHAT comes, not the order.\n", " Learnability rests on the empirical rank-10 occupancy result.\n", " Lit. Sampled softmax requires the log-Q correction; with a uniform\n", " proposal the correction is constant and cancels in the softmax\n", " (target always included). head='sampled' implements exactly this.\n", "\n", "THE BRANCHING GAUGE (the [TAU] kernel invariant, inverted): a single unit row\n", "has conf = ||(p+ - p-)A|| pinned at f(tau,K,D). A PREDICTED pi is not so\n", "bound — implied confidence below the invariant is the model declaring\n", "superposition. branching_frac is monitored from step zero.\n", "\n", "Banks: 'corpus' (top-M trigrams of the training stream), 'wordnet'\n", "(AbstractPhil/wordnet-lexical-topology char_eng_3gram, frequency-ranked,\n", "filtered to exact 3-byte UTF-8), or per-step 'sampled' negatives.\n", "\n", "Usage (Blackwell / A100):\n", " try:\n", " from aleph_lm import AlephLMConfig, train_aleph_lm\n", " except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", " r = train_aleph_lm(AlephLMConfig(steps=10_000, device='cuda',\n", " head='hybrid', bank_source='wordnet'))\n", "\n", "Depends: aleph_routed_attention.py, aleph_trigram_lm.py in the same directory.\n", "Author: AbstractPhil + Mirel Date: 2026-06-09 License: MIT\n", "\"\"\"\n", "\n", "from __future__ import annotations\n", "\n", "import math\n", "import os\n", "import time\n", "from dataclasses import dataclass\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import numpy as np\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from aleph_routed_attention import AlephRoutedAttention, AlephAttentionConfig\n", "except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", "try:\n", " from aleph_trigram_lm import TrigramStream, statute\n", "except ImportError:\n", " pass # pasted in-namespace (Colab cells)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Config\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class AlephLMConfig:\n", " # substrate\n", " corpus_id: str = \"wikitext-103-raw-v1\"\n", " split: str = \"train\"\n", " max_corpus_bytes: Optional[int] = 100_000_000\n", " seq_len: int = 256 # trigrams (3*seq_len bytes)\n", " seed: int = 1234\n", "\n", " # tower\n", " dim: int = 384\n", " n_layers: int = 4\n", " n_heads: int = 6\n", " K: int = 64\n", " D_addr: int = 4\n", " tau: float = 0.1\n", " codebook_init: object = \"random\"\n", " shared_codebook: bool = True\n", "\n", " # prediction head (the ledger's resolutions)\n", " head: str = \"hybrid\" # 'hybrid' | 'byte' | 'bank' | 'sampled'\n", " pi_mode: str = \"free\" # 'free' (T2 default) | 'address' (unimodal ablation)\n", " bank_source: str = \"corpus\" # 'corpus' | 'wordnet'\n", " bank_size: int = 4096\n", " n_negatives: int = 1024 # head='sampled'\n", " logit_scale_init: float = 1.0 # alpha on the log-kernel logits (T6)\n", "\n", " # hybrid bank scorer: 'kernel' (log-kernel, T6) or 'pmix' — a mixture of\n", " # J pointers on S^(d_point-1): logits(c) = logsumexp_j [log w_j + T yhat_j.c]\n", " # = Mixture-of-Softmaxes in sphere coordinates. Theorem-backed twice over:\n", " # raises output rank past the T7 bottleneck (MoS, Yang et al.), and gives\n", " # the pointer J modes so the barycenter pathology (unimodal aim at a\n", " # multimodal future) is structurally removed. Full-bank softmax retained:\n", " # T5 propriety intact. PREREGISTERED statute prediction: pmix candidate\n", " # coords bypass the codebook (W_cand48), removing prediction-side\n", " # discrimination pressure -> expect dev near the zero group, vs kernel's\n", " # +0.013. The dose-response gets a within-architecture test.\n", " # 'apmix' (RESTORATION, 2026-06-12): the audit found pmix bypasses the\n", " # codebook entirely (prediction-side aleph gone; ablating the trained\n", " # codebook cost +0.009 bpb — the routing learned to ignore it, and the\n", " # collapse followed from unemployment). apmix makes the mixture head and\n", " # the aleph the SAME mechanism: candidate coords = sqrt of the signed\n", " # codebook address (Hellinger embedding of kappa: prob vector on the 2K\n", " # simplex -> unit point on S^(2K-1); pointer.coord = Bhattacharyya\n", " # affinity). Full-bank softmax kept (T5), MoS rank escape kept (T7),\n", " # and full discrimination pressure returns to the codebook (+hemisphere).\n", " bank_scorer: str = \"kernel\" # 'kernel' | 'pmix' | 'apmix'\n", " # ── Tier-A scaling switches (2026-06-11) ──\n", " # bank_softmax='sampled' (pmix only): train CE over {batch targets} ∪\n", " # n_bank_samples uniform negatives with the log-Q correction (Jean et al.;\n", " # shared-negative approximation documented at the call site). The REPORTED\n", " # bpb stays honest: at every log step the in-bank NLL is recomputed against\n", " # the FULL bank in chunked no_grad. pos_mode='clamp' saturates position\n", " # indices at seq_len-1 -> unbounded streaming/generation (decay-gated state\n", " # is the principled v2). train_mode='stream' = TBPTT-1 over `segments`\n", " # carried streaming states: effective context segments*seq_len at constant\n", " # memory (requires pos_mode='clamp').\n", " bank_softmax: str = \"full\" # 'full' | 'sampled'\n", " n_bank_samples: int = 8192\n", " pos_mode: str = \"absolute\" # 'absolute' | 'clamp'\n", " train_mode: str = \"window\" # 'window' | 'stream'\n", " segments: int = 4\n", " amp: bool = True # bf16 autocast on CUDA\n", " ckpt_every: int = 0 # save checkpoint every N steps (0 = end only) — cull insurance\n", " hub_repo: Optional[str] = None # e.g. 'AbstractPhil/geolip-aleph-void' — every saved checkpoint\n", " hub_dir: str = \"experiments/exp_008_tier_a\" # also uploads here (non-fatal on failure); snapshots at end\n", " init_from: Optional[str] = None # warm-start model weights from a checkpoint (fresh optimizer; not an exact resume — honest note)\n", " compile_backbone: bool = False # compile backbone only (tensor out)\n", " # ── codebook preservation (post-degeneracy findings, 2026-06-12) ──\n", " # Three strategies against routing-driven codebook collapse (exp_008:\n", " # eff.rank 3.66 -> 1.89 by step 6000 under streamed long-context routing):\n", " # freeze_codebook='off' free codebook (status quo)\n", " # freeze_codebook='spread' FROZEN deterministic maximal-spread points\n", " # (repulsion-optimized on S^(Da-1), seeded) —\n", " # statute by construction; routing reshapes\n", " # queries, not axes. The geometry-primary arm.\n", " # freeze_codebook='init' freeze whatever the seed initialized\n", " # codebook_lr_mult: timescale arm — codebook param group at lr*mult\n", " # (e.g. 0.05): collapse delayed past the training horizon, not prevented.\n", " # div_weight (existing): counter-force arm. exp_009 arbitrates all three.\n", " freeze_codebook: str = \"off\" # 'off' | 'spread' | 'init'\n", " codebook_lr_mult: float = 1.0\n", " kernel_aux_weight: float = 0.0 # restoration arm B: small kernel-scorer CE through the codebook riding alongside pmix (re-employs the codebook without changing the main head)\n", " n_pointers: int = 4 # J mixture components (pmix)\n", "\n", " # pointer head (head='pointer'): NN-on-the-sphere decode\n", " d_point: int = 48 # pointer sphere dim (band-valid; the\n", " # capacity table gives the decode\n", " # budget theta_NN/2 at this D)\n", " pointer_k: int = 32 # hard negatives = target's k sphere-NN\n", " pointer_cos_weight: float = 0.5 # aiming regularizer (contrastive CE\n", " # is the main learner — lit. caveat)\n", " pointer_refresh: int = 200 # steps between NN-table refreshes\n", " # (candidate coords drift)\n", "\n", " # write-head (T8, auxiliary multiset prediction)\n", " write_weight: float = 0.1 # 0 disables\n", " write_horizon: int = 8 # W: the granularity dial\n", "\n", " # training\n", " steps: int = 10_000\n", " batch_size: int = 32\n", " accum_steps: int = 1\n", " lr: float = 3e-4\n", " lr_decay: bool = True\n", " div_weight: float = 0.0\n", " log_every: int = 250\n", " device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "\n", " # outputs\n", " snapshot_codebook: bool = True\n", " snapshot_path: str = \"aleph_lm_snaps.pt\"\n", " checkpoint_path: Optional[str] = \"aleph_lm.pt\"\n", "\n", " def __post_init__(self):\n", " assert self.head in (\"hybrid\", \"byte\", \"bank\", \"sampled\", \"pointer\")\n", " assert self.pi_mode in (\"free\", \"address\")\n", " assert self.bank_scorer in (\"kernel\", \"pmix\", \"apmix\")\n", " assert self.bank_softmax in (\"full\", \"sampled\")\n", " assert self.pos_mode in (\"absolute\", \"clamp\")\n", " assert self.train_mode in (\"window\", \"stream\")\n", " if self.bank_softmax == \"sampled\":\n", " assert self.head == \"hybrid\" and self.bank_scorer in (\"pmix\", \"apmix\"), \\\n", " \"sampled softmax is wired for the pmix/apmix hybrid\"\n", " if self.train_mode == \"stream\":\n", " assert self.pos_mode == \"clamp\", \"stream training needs pos_mode='clamp'\"\n", " assert self.freeze_codebook in (\"off\", \"spread\", \"init\")\n", " assert self.bank_source in (\"corpus\", \"wordnet\") \\\n", " or os.path.isfile(str(self.bank_source)), \\\n", " f\"bank_source must be 'corpus'|'wordnet'|path to bank .pt\"\n", " assert self.dim % self.n_heads == 0\n", " assert self.write_horizon >= 1\n", " tag = self.head\n", " if self.head in (\"hybrid\", \"bank\"):\n", " b = (os.path.splitext(os.path.basename(str(self.bank_source)))[0]\n", " if os.path.isfile(str(self.bank_source)) else self.bank_source)\n", " tag += f\"_{b}\"\n", " if self.pi_mode != \"free\":\n", " tag += f\"_{self.pi_mode}\"\n", " if self.head == \"hybrid\" and self.bank_scorer in (\"pmix\", \"apmix\"):\n", " tag += f\"_{self.bank_scorer}{self.n_pointers}\"\n", " if self.checkpoint_path == \"aleph_lm.pt\":\n", " self.checkpoint_path = f\"aleph_lm_{tag}.pt\"\n", " if self.snapshot_path == \"aleph_lm_snaps.pt\":\n", " self.snapshot_path = f\"aleph_lm_snaps_{tag}.pt\"\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Candidate banks\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _tri_ids(tri: Tensor) -> Tensor:\n", " \"\"\"(..., 3) bytes -> scalar trigram id in [0, 256^3).\"\"\"\n", " return tri[..., 0] * 65536 + tri[..., 1] * 256 + tri[..., 2]\n", "\n", "\n", "def build_corpus_bank(stream: TrigramStream, M: int,\n", " sample_bytes: int = 6_000_000) -> Tensor:\n", " \"\"\"Top-M most frequent trigrams of the training stream. (M, 3) long.\"\"\"\n", " n = min(sample_bytes, (len(stream.stream) // 3) * 3)\n", " tri = stream.stream[:n].reshape(-1, 3)\n", " ids = (tri[:, 0].astype(np.int64) * 65536 + tri[:, 1].astype(np.int64) * 256\n", " + tri[:, 2].astype(np.int64))\n", " uniq, counts = np.unique(ids, return_counts=True)\n", " top = uniq[np.argsort(counts)[::-1][:M]]\n", " out = np.stack([top // 65536, (top // 256) % 256, top % 256], axis=-1)\n", " return torch.from_numpy(out.astype(np.int64))\n", "\n", "\n", "def build_wordnet_bank(M: int) -> Tensor:\n", " \"\"\"char_eng_3gram from AbstractPhil/wordnet-lexical-topology, frequency-\n", " ranked, filtered to exact 3-byte UTF-8. (M', 3) long, M' <= M.\"\"\"\n", " from huggingface_hub import hf_hub_download\n", " import pyarrow.parquet as pq\n", " p = hf_hub_download(\"AbstractPhil/wordnet-lexical-topology\",\n", " \"data/char_eng_3gram-00000-of-00001.parquet\",\n", " repo_type=\"dataset\")\n", " t = pq.read_table(p, columns=[\"ngram\", \"rank\"]).to_pandas()\n", " t = t.sort_values(\"rank\")\n", " rows = []\n", " for s in t[\"ngram\"]:\n", " b = str(s).encode(\"utf-8\", errors=\"ignore\")\n", " if len(b) == 3:\n", " rows.append([b[0], b[1], b[2]])\n", " if len(rows) >= M:\n", " break\n", " assert rows, \"wordnet bank empty after 3-byte filter\"\n", " return torch.tensor(rows, dtype=torch.long)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Model\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class AlephLM(nn.Module):\n", " \"\"\"The composite reduction. forward_loss(ids, targets) -> (loss, logs).\"\"\"\n", "\n", " def __init__(self, cfg: AlephLMConfig, bank: Optional[Tensor] = None):\n", " super().__init__()\n", " self.cfg = cfg\n", " d = cfg.dim\n", "\n", " # ── EMBED (byte-factored; shared with candidate composition) ──\n", " self.byte_emb = nn.ModuleList([nn.Embedding(256, d) for _ in range(3)])\n", " self.pos = nn.Parameter(0.02 * torch.randn(1, cfg.seq_len, d))\n", "\n", " # ── MIX (hub tower, shared codebook) ──\n", " def make_attn():\n", " return AlephRoutedAttention(AlephAttentionConfig(\n", " dim=d, num_heads=cfg.n_heads, mode=\"hub\", K=cfg.K,\n", " D_addr=cfg.D_addr, tau=cfg.tau, causal=True,\n", " codebook_init=cfg.codebook_init))\n", " self.layers = nn.ModuleList([\n", " nn.ModuleDict({\"norm1\": nn.LayerNorm(d), \"attn\": make_attn(),\n", " \"norm2\": nn.LayerNorm(d),\n", " \"mlp\": nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(),\n", " nn.Linear(4 * d, d))})\n", " for _ in range(cfg.n_layers)])\n", " if cfg.shared_codebook:\n", " shared = self.layers[0][\"attn\"].codebook\n", " for L in self.layers[1:]:\n", " L[\"attn\"].codebook = shared\n", " self.norm_f = nn.LayerNorm(d)\n", "\n", " # ── PREDICT: pi over 2K oriented axes ──\n", " self.W_pi = nn.Linear(d, cfg.K, bias=True) # tied logits [w; -w]\n", " if cfg.pi_mode == \"address\": # unimodal ablation (T2)\n", " self.W_pi_row = nn.Linear(d, cfg.D_addr, bias=False)\n", " nn.init.orthogonal_(self.W_pi_row.weight)\n", "\n", " # ── CANDIDATE: compositional addresses (banked heads) ──\n", " self.W_kappa = nn.Linear(d, cfg.D_addr, bias=False)\n", " nn.init.orthogonal_(self.W_kappa.weight)\n", " self.logit_scale = nn.Parameter(\n", " torch.tensor(float(cfg.logit_scale_init))) # alpha (T6)\n", "\n", " # ── byte-product head (T4 floor + hybrid tail) ──\n", " self.byte_heads = nn.ModuleList([nn.Linear(d, 256) for _ in range(3)])\n", "\n", " # ── hybrid gate (T5) ──\n", " self.gate = nn.Linear(d, 1)\n", "\n", " # ── pmix bank scorer: J-pointer mixture (MoS on the sphere) ──\n", " if cfg.head == \"hybrid\" and cfg.bank_scorer in (\"pmix\", \"apmix\"):\n", " J = cfg.n_pointers\n", " self._dp = cfg.d_point if cfg.bank_scorer == \"pmix\" else 2 * cfg.K\n", " self.W_pmix = nn.Linear(d, J * self._dp, bias=False)\n", " nn.init.orthogonal_(self.W_pmix.weight)\n", " self.W_mixgate = nn.Linear(d, J)\n", " if cfg.bank_scorer == \"pmix\":\n", " self.W_cand48 = nn.Linear(d, cfg.d_point, bias=False)\n", " nn.init.orthogonal_(self.W_cand48.weight)\n", " self.point_T = nn.Parameter(torch.tensor(10.0))\n", "\n", " # ── pointer head: predict a point on S^(d_point-1), decode by NN ──\n", " if cfg.head == \"pointer\":\n", " self.W_point = nn.Linear(d, cfg.d_point, bias=False)\n", " nn.init.orthogonal_(self.W_point.weight)\n", " self.W_cand48 = nn.Linear(d, cfg.d_point, bias=False)\n", " nn.init.orthogonal_(self.W_cand48.weight)\n", " self.point_T = nn.Parameter(torch.tensor(10.0)) # contrastive inv-temp\n", " self.register_buffer(\"_nn_table\", torch.zeros(0, dtype=torch.long),\n", " persistent=False)\n", " self._nn_step = -1\n", "\n", " # ── write-head (T8): predicted Delta-z over 2K ──\n", " if cfg.write_weight > 0:\n", " self.W_write = nn.Linear(d, 2 * cfg.K)\n", "\n", " # ── bank registration ──\n", " if bank is not None:\n", " self.register_buffer(\"bank\", bank) # (M, 3)\n", " self.register_buffer(\"bank_ids_sorted\",\n", " _tri_ids(bank).sort().values) # membership\n", " self.register_buffer(\"bank_perm\",\n", " _tri_ids(bank).argsort()) # sorted->orig\n", " else:\n", " self.bank = None\n", "\n", " # ---- shared codebook handle ----\n", " @property\n", " def codebook(self) -> Tensor:\n", " return self.layers[0][\"attn\"].codebook\n", "\n", " def aleph_layers(self) -> List[AlephRoutedAttention]:\n", " return [m for m in self.modules() if isinstance(m, AlephRoutedAttention)]\n", "\n", " # ---- address of arbitrary unit rows vs the SHARED codebook ----\n", " def _address_rows(self, rows: Tensor) -> Tuple[Tensor, Tensor]:\n", " A = F.normalize(self.codebook, dim=-1)\n", " u = (rows @ A.t()) / self.cfg.tau\n", " m = u.abs().amax(-1, keepdim=True)\n", " ep, en = torch.exp(u - m), torch.exp(-u - m)\n", " Z = (ep + en).sum(-1, keepdim=True)\n", " return ep / Z, en / Z\n", "\n", " # ---- PREDICT ----\n", " def _pi(self, h: Tensor) -> Tuple[Tensor, Tensor]:\n", " \"\"\"pi over 2K oriented axes. 'free': tied simplex (T2 default).\n", " 'address': unimodal ablation.\"\"\"\n", " if self.cfg.pi_mode == \"address\":\n", " row = F.normalize(self.W_pi_row(h), dim=-1)\n", " return self._address_rows(row)\n", " w = self.W_pi(h) # (..., K)\n", " m = w.abs().amax(-1, keepdim=True)\n", " ep, en = torch.exp(w - m), torch.exp(-w - m)\n", " Z = (ep + en).sum(-1, keepdim=True)\n", " return ep / Z, en / Z\n", "\n", " # ---- CANDIDATE addresses for a (M, 3) byte bank ----\n", " def _kappa(self, bank: Tensor) -> Tuple[Tensor, Tensor]:\n", " e = sum(emb(bank[:, i]) for i, emb in enumerate(self.byte_emb))\n", " rows = F.normalize(self.W_kappa(e), dim=-1) # (M, D_addr)\n", " return self._address_rows(rows)\n", "\n", " # ---- SCORE: log-kernel logits (T6) ----\n", " def _bank_logits(self, pi_p: Tensor, pi_m: Tensor,\n", " k_p: Tensor, k_m: Tensor) -> Tensor:\n", " s = pi_p @ k_p.t() + pi_m @ k_m.t() # (..., M), > 0\n", " return self.logit_scale * torch.log(s.clamp_min(1e-9))\n", "\n", " # ---- pmix bank logits: logsumexp over J pointers (MoS on the sphere) ----\n", " def _pmix_logits(self, h: Tensor,\n", " cols: Optional[Tensor] = None) -> Tuple[Tensor, Dict[str, float]]:\n", " cfg = self.cfg\n", " J = cfg.n_pointers\n", " if cfg.bank_scorer == \"apmix\":\n", " bank = self.bank if cols is None else self.bank[cols]\n", " k_p, k_m = self._kappa(bank) # (Mc,K) each, sum=1\n", " coords = torch.sqrt(torch.cat([k_p, k_m], dim=-1)\n", " .clamp_min(1e-9)) # Hellinger: S^(2K-1)\n", " else:\n", " coords = self._cand_coords() # (M, d_point)\n", " if cols is not None:\n", " coords = coords[cols] # (Mc, d_point)\n", " y = self.W_pmix(h).view(*h.shape[:-1], J, self._dp)\n", " y = F.normalize(y, dim=-1) # (B,S,J,dp)\n", " mix = F.log_softmax(self.W_mixgate(h), dim=-1) # (B,S,J)\n", " sims = torch.einsum(\"bsjd,md->bsjm\", y, coords) * self.point_T\n", " logits = torch.logsumexp(mix.unsqueeze(-1) + sims, dim=2) # (B,S,M)\n", " with torch.no_grad(): # mode diagnostics\n", " pw = torch.einsum(\"bsjd,bskd->bsjk\", y, y)\n", " off = pw.masked_select(~torch.eye(J, dtype=torch.bool,\n", " device=h.device)\n", " .expand_as(pw)).clamp(-1, 1)\n", " spread = torch.acos(off).mean().item() * 180 / math.pi\n", " usage = mix.exp().mean(dim=(0, 1))\n", " ent = -(usage * usage.clamp_min(1e-9).log()).sum().item() / math.log(J)\n", " return logits, {\"mode_spread_deg\": spread, \"mix_entropy\": ent}\n", "\n", " # ---- position (absolute, or clamped for unbounded streaming) ----\n", " def _pos_slice(self, S: int, offset: int = 0) -> Tensor:\n", " if self.cfg.pos_mode == \"clamp\":\n", " idx = (torch.arange(S, device=self.pos.device) + offset\n", " ).clamp_max(self.cfg.seq_len - 1)\n", " return self.pos[:, idx]\n", " return self.pos[:, offset: offset + S]\n", "\n", " # ---- honest full-bank in-bank NLL (chunked, no_grad) for sampled mode ----\n", " @torch.no_grad()\n", " def _pmix_full_nll(self, h: Tensor, bidx: Tensor, in_bank: Tensor) -> Tensor:\n", " M = self.bank.shape[0]\n", " lse = None\n", " tgt_logit = torch.zeros_like(bidx, dtype=h.dtype)\n", " for lo in range(0, M, 8192):\n", " cols = torch.arange(lo, min(lo + 8192, M), device=h.device)\n", " lg, _ = self._pmix_logits(h, cols=cols) # (B,S,Mc)\n", " chunk_lse = torch.logsumexp(lg, dim=-1)\n", " lse = chunk_lse if lse is None else torch.logaddexp(lse, chunk_lse)\n", " hit = in_bank & (bidx >= lo) & (bidx < lo + cols.numel())\n", " if hit.any():\n", " tgt_logit[hit] = lg[hit].gather(\n", " -1, (bidx[hit] - lo).unsqueeze(-1)).squeeze(-1)\n", " return lse - tgt_logit # NLL (B,S)\n", "\n", " # ---- tower ----\n", " def backbone(self, ids: Tensor) -> Tensor:\n", " x = sum(emb(ids[..., i]) for i, emb in enumerate(self.byte_emb))\n", " x = x + self._pos_slice(ids.shape[1])\n", " for L in self.layers:\n", " x = x + L[\"attn\"](L[\"norm1\"](x))\n", " x = x + L[\"mlp\"](L[\"norm2\"](x))\n", " return self.norm_f(x)\n", "\n", " # ---- byte-product log-probs of given targets (T4 tail) ----\n", " def _byte_logprob(self, h: Tensor, targets: Tensor) -> Tensor:\n", " lp = 0.0\n", " for c, head in enumerate(self.byte_heads):\n", " lp = lp + F.log_softmax(head(h), dim=-1).gather(\n", " -1, targets[..., c:c + 1]).squeeze(-1)\n", " return lp # (B, S)\n", "\n", " # ---- bank membership: target -> bank index or -1 ----\n", " def _bank_index(self, targets: Tensor) -> Tensor:\n", " tid = _tri_ids(targets)\n", " pos = torch.searchsorted(self.bank_ids_sorted, tid)\n", " pos = pos.clamp_max(len(self.bank_ids_sorted) - 1)\n", " hit = self.bank_ids_sorted[pos] == tid\n", " idx = self.bank_perm[pos]\n", " return torch.where(hit, idx, torch.full_like(idx, -1))\n", "\n", " # ---- write-head target (T8): order-marginalized future address mass ----\n", " @torch.no_grad()\n", " def _write_target(self, ids: Tensor) -> Tensor:\n", " \"\"\"Delta-z over 2K for horizon W at each position (normalized).\"\"\"\n", " cfg = self.cfg\n", " a0 = self.layers[0][\"attn\"]\n", " x = sum(emb(ids[..., i]) for i, emb in enumerate(self.byte_emb))\n", " x = x + self.pos[:, : ids.shape[1]]\n", " kh = a0._split_addr(a0.k_addr(self.layers[0][\"norm1\"](x)),\n", " ids.shape[0], ids.shape[1])\n", " pk_p, pk_m = a0._address(kh) # (B,H,S,K)\n", " p = torch.cat([pk_p, pk_m], dim=-1).mean(dim=1) # (B,S,2K)\n", " cs = torch.cat([torch.zeros_like(p[:, :1]), p.cumsum(dim=1)], dim=1)\n", " W = cfg.write_horizon\n", " B, S, _ = p.shape\n", " end = torch.arange(S, device=p.device).clamp_max(S - 1)\n", " lo = cs[:, 1:] # prefix up to t (incl)\n", " hi = cs[:, torch.clamp(torch.arange(S, device=p.device) + W, max=S)]\n", " dz = (hi - lo).clamp_min(0)\n", " valid = (torch.arange(S, device=p.device) + 1 < S) # at least 1 future tok\n", " dz = dz / dz.sum(-1, keepdim=True).clamp_min(1e-9)\n", " return dz, valid\n", "\n", " # ---- pointer head: compositional D=48 candidate coordinates ----\n", " def _cand_coords(self) -> Tensor:\n", " e = sum(emb(self.bank[:, i]) for i, emb in enumerate(self.byte_emb))\n", " return F.normalize(self.W_cand48(e), dim=-1) # (M, d_point)\n", "\n", " @torch.no_grad()\n", " def _refresh_nn(self, coords: Tensor, step: int) -> None:\n", " \"\"\"Hard-negative table: each candidate's k nearest sphere neighbors\n", " (excluding self). Refreshed periodically — coordinates drift.\"\"\"\n", " cos = coords @ coords.t()\n", " cos.fill_diagonal_(-2.0)\n", " self._nn_table = cos.topk(self.cfg.pointer_k, dim=-1).indices # (M, k)\n", " self._nn_step = step\n", " # decode budget: theta_NN/2 of the CURRENT candidate constellation\n", " nn_deg = torch.acos(cos.max(dim=-1).values.clamp(-1, 1)) * 180 / math.pi\n", " self._decode_budget_deg = (nn_deg.median() / 2).item()\n", "\n", " def _pointer_loss(self, h: Tensor, targets: Tensor,\n", " step: int) -> Tuple[Tensor, Dict]:\n", " \"\"\"NN-on-the-sphere head (T5-chained with the byte tail):\n", " in-bank: -log gate - log softmax_{target ∪ kNN(target)}(T * yhat·c)\n", " + lambda_cos (1 - yhat·c_target) [aiming term]\n", " out-bank: -log(1-gate) - log P_byte(g)\n", " Decode metric: exact-NN rate + median angular error vs the budget\n", " theta_NN/2 (the decode-correctness theorem).\"\"\"\n", " cfg = self.cfg\n", " logs: Dict[str, float] = {}\n", " coords = self._cand_coords() # (M, d_point)\n", " if step - self._nn_step >= cfg.pointer_refresh or len(self._nn_table) == 0:\n", " self._refresh_nn(coords.detach(), step)\n", "\n", " yhat = F.normalize(self.W_point(h), dim=-1) # (B,S,d_point)\n", " bidx = self._bank_index(targets)\n", " in_bank = bidx >= 0\n", " logs[\"coverage\"] = in_bank.float().mean().item()\n", "\n", " g_logit = self.gate(h).squeeze(-1)\n", " nll_byte = -self._byte_logprob(h, targets)\n", "\n", " B, S = bidx.shape\n", " tgt = bidx.clamp_min(0) # (B,S)\n", " negs = self._nn_table[tgt] # (B,S,k) hard negatives\n", " cand_idx = torch.cat([tgt.unsqueeze(-1), negs], dim=-1) # (B,S,1+k)\n", " c = coords[cand_idx] # (B,S,1+k,d_point)\n", " logits = torch.einsum(\"bsd,bsnd->bsn\", yhat, c) * self.point_T\n", " nll_point = F.cross_entropy(\n", " logits.reshape(-1, logits.shape[-1]),\n", " torch.zeros(B * S, dtype=torch.long, device=h.device),\n", " reduction=\"none\").view(B, S)\n", " cos_t = torch.einsum(\"bsd,bsd->bs\", yhat, coords[tgt])\n", " aim = cfg.pointer_cos_weight * (1.0 - cos_t)\n", "\n", " nll = torch.where(in_bank,\n", " -F.logsigmoid(g_logit) + nll_point + aim,\n", " -F.logsigmoid(-g_logit) + nll_byte)\n", " loss = nll.mean()\n", " logs[\"bpb\"] = loss.item() / 3 / math.log(2)\n", " logs[\"gate_acc\"] = ((torch.sigmoid(g_logit) > 0.5) == in_bank\n", " ).float().mean().item()\n", " with torch.no_grad(): # decode metrics\n", " if in_bank.any():\n", " full = (yhat @ coords.t()) # (B,S,M)\n", " pred = full.argmax(-1)\n", " logs[\"nn_exact\"] = (pred[in_bank] == tgt[in_bank]\n", " ).float().mean().item()\n", " # PROPER eval likelihood: full-bank softmax (comparable to\n", " # hybrid bpb; the training loss above is contrastive-over-33\n", " # and is NOT a likelihood — do not compare it across heads)\n", " nll_full = F.cross_entropy(\n", " (full * self.point_T).reshape(-1, full.shape[-1]),\n", " tgt.reshape(-1), reduction=\"none\").view_as(tgt)\n", " nll_eval = torch.where(in_bank,\n", " -F.logsigmoid(g_logit) + nll_full,\n", " -F.logsigmoid(-g_logit) + nll_byte)\n", " logs[\"bpb_eval\"] = nll_eval.mean().item() / 3 / math.log(2)\n", " ang = torch.acos(cos_t[in_bank].clamp(-1, 1)) * 180 / math.pi\n", " logs[\"ang_err_deg\"] = ang.median().item()\n", " logs[\"budget_deg\"] = self._decode_budget_deg\n", " logs[\"in_budget\"] = (ang < self._decode_budget_deg\n", " ).float().mean().item()\n", " return loss, logs\n", "\n", " # ---- the loss (T5-exact hybrid + auxiliaries) ----\n", " def forward_loss(self, ids: Tensor, targets: Tensor,\n", " step: int = 0, h: Optional[Tensor] = None\n", " ) -> Tuple[Tensor, Dict]:\n", " cfg = self.cfg\n", " if h is None:\n", " h = self.backbone(ids) # (B,S,d)\n", " logs: Dict[str, float] = {}\n", "\n", " if cfg.head == \"pointer\":\n", " assert self.bank is not None, \"pointer head requires a bank\"\n", " return self._pointer_loss(h, targets, step)\n", "\n", " if cfg.head == \"byte\":\n", " nll = -self._byte_logprob(h, targets) # (B,S)\n", " loss = nll.mean()\n", " logs[\"bpb\"] = loss.item() / 3 / math.log(2)\n", " return loss, logs\n", "\n", " pi_p, pi_m = self._pi(h) # (B,S,K) each\n", "\n", " if cfg.head == \"sampled\":\n", " # uniform negatives + target; uniform proposal => logQ constant,\n", " # cancels in softmax (literature requirement satisfied trivially)\n", " B, S, _ = h.shape\n", " neg = torch.randint(0, 256, (cfg.n_negatives, 3), device=h.device)\n", " cand = torch.cat([targets.reshape(-1, 3), neg], dim=0)\n", " cand_ids, inv = torch.unique(_tri_ids(cand), return_inverse=True)\n", " uniq = torch.stack([cand_ids // 65536, (cand_ids // 256) % 256,\n", " cand_ids % 256], dim=-1)\n", " k_p, k_m = self._kappa(uniq)\n", " logits = self._bank_logits(pi_p.reshape(-1, cfg.K),\n", " pi_m.reshape(-1, cfg.K), k_p, k_m)\n", " tgt_idx = inv[: B * S]\n", " loss = F.cross_entropy(logits, tgt_idx)\n", " logs[\"bpb\"] = loss.item() / 3 / math.log(2)\n", " logs[\"n_cand\"] = float(len(uniq))\n", " return loss, logs\n", "\n", " # banked heads\n", " assert self.bank is not None, \"head='hybrid'/'bank' requires a bank\"\n", " bidx = self._bank_index(targets) # (B,S), -1 = miss\n", " in_bank = bidx >= 0\n", " logs[\"coverage\"] = in_bank.float().mean().item()\n", " sampled = (cfg.bank_softmax == \"sampled\" and self.training)\n", " if cfg.head == \"hybrid\" and cfg.bank_scorer in (\"pmix\", \"apmix\"):\n", " if sampled:\n", " # shared-negative sampled softmax: cols = batch targets ∪\n", " # uniform negatives; log-Q correction log(n/M) on pure\n", " # negatives only (cols that are some example's target carry\n", " # Q≈1; tiny documented bias — reported bpb is recomputed\n", " # full-bank below at log steps, so the LOGGED number is exact)\n", " M = self.bank.shape[0]\n", " n = min(cfg.n_bank_samples, M)\n", " tcols = bidx[in_bank].unique()\n", " samp = torch.randint(0, M, (n,), device=h.device)\n", " cols = torch.unique(torch.cat([tcols, samp]))\n", " logits, pm_logs = self._pmix_logits(h, cols=cols)\n", " logits = logits + torch.where(\n", " torch.isin(cols, tcols), 0.0,\n", " -math.log(n / M)).to(logits.dtype) # -(-logQ)\n", " bidx = torch.searchsorted(cols, bidx.clamp_min(0))\n", " else:\n", " logits, pm_logs = self._pmix_logits(h) # (B,S,M)\n", " logs.update(pm_logs)\n", " else:\n", " k_p, k_m = self._kappa(self.bank) # (M,K) each\n", " logits = self._bank_logits(pi_p, pi_m, k_p, k_m) # (B,S,M)\n", "\n", " if cfg.head == \"bank\":\n", " # ablation head: proper only on covered targets (coverage logged)\n", " lb = F.log_softmax(logits, dim=-1)\n", " nll = -lb.gather(-1, bidx.clamp_min(0).unsqueeze(-1)).squeeze(-1)\n", " loss = nll[in_bank].mean() if in_bank.any() else logits.sum() * 0\n", " logs[\"bpb_inbank\"] = (loss.item() / 3 / math.log(2)\n", " if in_bank.any() else float(\"nan\"))\n", " return loss, logs\n", "\n", " # ── T5-exact hybrid: -log P(g) per position ──\n", " g_logit = self.gate(h).squeeze(-1) # (B,S)\n", " log_g = F.logsigmoid(g_logit)\n", " log_1mg = F.logsigmoid(-g_logit)\n", " lb = F.log_softmax(logits, dim=-1)\n", " nll_bank = -lb.gather(-1, bidx.clamp_min(0).unsqueeze(-1)).squeeze(-1)\n", " nll_byte = -self._byte_logprob(h, targets)\n", " nll = torch.where(in_bank, -log_g + nll_bank, -log_1mg + nll_byte)\n", " loss = nll.mean()\n", " lm_loss = loss.detach() # LM-only nll: keeps bpb honest under aux\n", " if (cfg.kernel_aux_weight > 0 and cfg.head == \"hybrid\"\n", " and cfg.bank_scorer in (\"pmix\", \"apmix\") and in_bank.any()):\n", " # restoration arm B: discrimination pressure THROUGH the codebook\n", " # (the rank-building (+) hemisphere), independent of the main head\n", " aux_bank = self.bank if not sampled else self.bank[cols]\n", " ak_p, ak_m = self._kappa(aux_bank)\n", " aux_logits = self._bank_logits(pi_p, pi_m, ak_p, ak_m)\n", " if sampled:\n", " aux_logits = aux_logits + torch.where(\n", " torch.isin(cols, tcols), 0.0,\n", " -math.log(n / M)).to(aux_logits.dtype)\n", " aux_lb = F.log_softmax(aux_logits, dim=-1)\n", " aux_nll = -aux_lb.gather(-1, bidx.clamp_min(0)\n", " .unsqueeze(-1)).squeeze(-1)\n", " kaux = aux_nll[in_bank].mean()\n", " loss = loss + cfg.kernel_aux_weight * kaux\n", " logs[\"kernel_aux\"] = kaux.item()\n", " if sampled and (step % max(cfg.log_every, 1) == 0):\n", " nb_full = self._pmix_full_nll(h.detach(), self._bank_index(targets),\n", " in_bank)\n", " nll_h = torch.where(in_bank, -log_g.detach() + nb_full,\n", " -log_1mg.detach() + nll_byte.detach())\n", " logs[\"bpb\"] = nll_h.mean().item() / 3 / math.log(2) # honest\n", " logs[\"bpb_s\"] = lm_loss.item() / 3 / math.log(2) # sampled LM\n", " else:\n", " logs[\"bpb\"] = lm_loss.item() / 3 / math.log(2) # LM only\n", " with torch.no_grad(): # branch-conditional currencies\n", " if in_bank.any():\n", " logs[\"bpb_bank_cond\"] = nll_bank[in_bank].mean().item() / 3 / math.log(2)\n", " if (~in_bank).any():\n", " logs[\"bpb_byte_cond\"] = nll_byte[~in_bank].mean().item() / 3 / math.log(2)\n", " logs[\"gate_acc\"] = ((torch.sigmoid(g_logit) > 0.5) == in_bank\n", " ).float().mean().item()\n", "\n", " # ── auxiliaries ──\n", " if cfg.write_weight > 0:\n", " dz, valid = self._write_target(ids)\n", " pred = F.log_softmax(self.W_write(h), dim=-1)\n", " kl = F.kl_div(pred, dz, reduction=\"none\").sum(-1)\n", " wl = kl[:, valid].mean()\n", " loss = loss + cfg.write_weight * wl\n", " logs[\"write_kl\"] = wl.item()\n", "\n", " return loss, logs\n", "\n", "\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # Branch-predictive generation (streaming, gauge-triggered forking)\n", " # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", " # The CPU metaphor, implemented: per-branch state is the streaming\n", " # (M±, z±) per layer — O(K·d) regardless of context, so speculation is\n", " # cheap. The model's own multimodality estimate is the branch-predictor\n", " # confidence bit: fork ONLY where the next-distribution is genuinely\n", " # multimodal (top-2 probability ratio above `fork_ratio`); run straight\n", " # through deterministic stretches at full pipeline speed.\n", " # Position embedding is absolute: generation is capped at cfg.seq_len\n", " # total positions (documented v1 limitation).\n", "\n", " def stream_backbone(self, ids: Tensor, states: Optional[List],\n", " pos_offset: int) -> Tuple[Tensor, List]:\n", " \"\"\"One streamed segment (B, S_seg, 3) with per-layer carried states.\"\"\"\n", " x = sum(emb(ids[..., i]) for i, emb in enumerate(self.byte_emb))\n", " x = x + self._pos_slice(ids.shape[1], pos_offset)\n", " states = states or [None] * len(self.layers)\n", " new_states: List = []\n", " for L, st in zip(self.layers, states):\n", " a, ns = L[\"attn\"].forward_stream(L[\"norm1\"](x), state=st)\n", " x = x + a\n", " x = x + L[\"mlp\"](L[\"norm2\"](x))\n", " new_states.append(ns)\n", " return self.norm_f(x), new_states\n", "\n", " @torch.no_grad()\n", " def next_distribution(self, h_last: Tensor) -> Tuple[Tensor, Tensor, Tensor]:\n", " \"\"\"(gate_prob, bank_probs (M,), byte_logprobs (3,256)) for one position.\n", " h_last: (1, d). Bank scorer follows cfg.bank_scorer.\"\"\"\n", " cfg = self.cfg\n", " g = torch.sigmoid(self.gate(h_last)).squeeze()\n", " if cfg.bank_scorer in (\"pmix\", \"apmix\"):\n", " logits, _ = self._pmix_logits(h_last.unsqueeze(0))\n", " bank_p = F.softmax(logits.squeeze(0).squeeze(0), dim=-1)\n", " else:\n", " pi_p, pi_m = self._pi(h_last)\n", " k_p, k_m = self._kappa(self.bank)\n", " bank_p = F.softmax(self._bank_logits(pi_p, pi_m, k_p, k_m\n", " ).squeeze(0), dim=-1)\n", " byte_lp = torch.stack([F.log_softmax(head(h_last).squeeze(0), dim=-1)\n", " for head in self.byte_heads]) # (3,256)\n", " return g, bank_p, byte_lp\n", "\n", " @torch.no_grad()\n", " def generate_tree(self, prompt: bytes, max_new: int = 24,\n", " beam: int = 8, fork_ratio: float = 0.35,\n", " fork_width: int = 3, device: str = \"cpu\",\n", " temperature: float = 0.0) -> List[Dict]:\n", " \"\"\"Branch-predictive decoding. Returns the surviving branches as\n", " [{'text', 'logp', 'forks'}], best first.\n", " fork_ratio: fork iff p2/p1 > ratio (the confidence bit);\n", " fork_width: children per fork; beam: global survivor cap.\"\"\"\n", " self.eval()\n", " cfg = self.cfg\n", " b = prompt[: 3 * (len(prompt) // 3)] or b\" \"\n", " ids = torch.frombuffer(bytearray(b), dtype=torch.uint8) \\\n", " .to(torch.long).view(1, -1, 3).to(device)\n", " h, states = self.stream_backbone(ids, None, 0)\n", " pos = ids.shape[1]\n", " Branch = lambda st, h_, lp, txt, forks: \\\n", " {\"states\": st, \"h\": h_, \"logp\": lp, \"bytes\": txt, \"forks\": forks}\n", " branches = [Branch(states, h[:, -1], 0.0, b\"\", 0)]\n", "\n", " bank_bytes = self.bank.cpu().numpy().astype(\"uint8\") \\\n", " if self.bank is not None else None\n", " for step in range(max_new):\n", " if cfg.pos_mode == \"absolute\" and pos + 1 > cfg.seq_len:\n", " break\n", " nxt: List[Dict] = []\n", " for br in branches:\n", " g, bank_p, byte_lp = self.next_distribution(br[\"h\"])\n", " # mixture distribution over candidate continuations:\n", " # in-bank candidates carry g*bank_p; the byte tail is\n", " # summarized by its argmax trigram carrying (1-g)*p_byte\n", " cand_p, cand_tri = [], []\n", " if bank_bytes is not None:\n", " top_p, top_i = bank_p.topk(min(fork_width + 1, len(bank_p)))\n", " for p, i in zip(top_p.tolist(), top_i.tolist()):\n", " cand_p.append(g.item() * p)\n", " cand_tri.append(bytes(bank_bytes[i]))\n", " by = byte_lp.argmax(-1)\n", " p_by = float(byte_lp.max(-1).values.sum().exp())\n", " cand_p.append((1 - g.item()) * p_by)\n", " cand_tri.append(bytes(by.tolist()))\n", " order = np.argsort(cand_p)[::-1]\n", " p1 = cand_p[order[0]]\n", " p2 = cand_p[order[1]] if len(order) > 1 else 0.0\n", " forking = p1 > 0 and p2 / max(p1, 1e-12) > fork_ratio\n", " if temperature > 0: # sampled children: breaks\n", " pp = np.asarray(cand_p) ** (1.0 / temperature) # greedy loops\n", " pp = pp / pp.sum()\n", " k = min(fork_width if forking else 1, (pp > 0).sum())\n", " take = np.random.choice(len(pp), size=k, replace=False, p=pp)\n", " else:\n", " take = order[: fork_width] if forking else order[:1]\n", " forked = len(take) > 1\n", " for oi in take:\n", " tri = cand_tri[oi]\n", " t = torch.tensor(list(tri), dtype=torch.long,\n", " device=device).view(1, 1, 3)\n", " h2, st2 = self.stream_backbone(\n", " t, [tuple(s.clone() for s in st)\n", " for st in br[\"states\"]], pos)\n", " nxt.append(Branch(st2, h2[:, -1],\n", " br[\"logp\"] + math.log(max(cand_p[oi], 1e-12)),\n", " br[\"bytes\"] + tri,\n", " br[\"forks\"] + int(forked)))\n", " nxt.sort(key=lambda d: -d[\"logp\"])\n", " branches = nxt[:beam]\n", " pos += 1\n", " return [{\"text\": (prompt + br[\"bytes\"]).decode(\"utf-8\", errors=\"replace\"),\n", " \"logp\": br[\"logp\"], \"forks\": br[\"forks\"]}\n", " for br in branches]\n", "\n", " # ---- the branching gauge ([TAU] inverted) ----\n", " @torch.no_grad()\n", " def branching_gauge(self, ids: Tensor, n_baseline: int = 4096) -> Dict:\n", " h = self.backbone(ids)\n", " pi_p, pi_m = self._pi(h)\n", " A = F.normalize(self.codebook, dim=-1)\n", " conf = ((pi_p - pi_m) @ A).norm(dim=-1).reshape(-1)\n", " rows = F.normalize(torch.randn(n_baseline, self.cfg.D_addr,\n", " device=h.device), dim=-1)\n", " bp, bm = self._address_rows(rows)\n", " base = ((bp - bm) @ A).norm(dim=-1)\n", " mu, sd = base.mean(), base.std()\n", " return {\"conf_mean\": conf.mean().item(),\n", " \"kernel_invariant\": mu.item(),\n", " \"branching_frac\": (conf < mu - 2 * sd).float().mean().item()}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Training\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def train_aleph_lm(cfg: AlephLMConfig,\n", " stream: Optional[TrigramStream] = None) -> Dict:\n", " torch.manual_seed(cfg.seed)\n", " dev = torch.device(cfg.device)\n", " stream = stream or TrigramStream(cfg.corpus_id, cfg.split,\n", " cfg.max_corpus_bytes, cfg.seed)\n", " bank = None\n", " if cfg.head in (\"hybrid\", \"bank\", \"pointer\"):\n", " if os.path.isfile(str(cfg.bank_source)): # stratified-atlas bank\n", " d = torch.load(cfg.bank_source, map_location=\"cpu\", weights_only=False)\n", " bank = d[\"bank\"] if isinstance(d, dict) else d\n", " print(f\"[bank] loaded {len(bank)} trigram candidates \"\n", " f\"from {cfg.bank_source}\")\n", " elif cfg.bank_source == \"wordnet\":\n", " try:\n", " bank = build_wordnet_bank(cfg.bank_size)\n", " print(f\"[bank] wordnet char_eng_3gram: {len(bank)} types\")\n", " except Exception as e:\n", " print(f\"[bank] wordnet unavailable ({e}); falling back to corpus\")\n", " if bank is None:\n", " bank = build_corpus_bank(stream, cfg.bank_size)\n", " print(f\"[bank] corpus top-{len(bank)} trigrams\")\n", "\n", " model = AlephLM(cfg, bank=bank).to(dev)\n", " n_params = sum(p.numel() for p in model.parameters())\n", " # codebook preservation arms\n", " cb_params = list({id(a.codebook): a.codebook\n", " for a in model.aleph_layers()}.values()) # dedupe shared\n", " cb_ids = {id(p) for p in cb_params}\n", " if cfg.freeze_codebook != \"off\":\n", " if cfg.freeze_codebook == \"spread\":\n", " spread = _max_spread_points(cb_params[0].shape[0],\n", " cb_params[0].shape[1], seed=cfg.seed)\n", " with torch.no_grad():\n", " for p in cb_params:\n", " p.copy_(spread.to(p.device, p.dtype))\n", " for p in cb_params:\n", " p.requires_grad_(False)\n", " print(f\"[codebook] FROZEN ({cfg.freeze_codebook})\")\n", " opt = torch.optim.Adam([p for p in model.parameters()\n", " if p.requires_grad], lr=cfg.lr) # pure Adam\n", " elif cfg.codebook_lr_mult != 1.0:\n", " rest = [p for p in model.parameters() if id(p) not in cb_ids]\n", " opt = torch.optim.Adam(\n", " [{\"params\": rest, \"lr\": cfg.lr},\n", " {\"params\": cb_params, \"lr\": cfg.lr * cfg.codebook_lr_mult}],\n", " lr=cfg.lr) # pure Adam\n", " print(f\"[codebook] slow lane: lr x {cfg.codebook_lr_mult}\")\n", " else:\n", " opt = torch.optim.Adam(model.parameters(), lr=cfg.lr) # pure Adam\n", " sched = (torch.optim.lr_scheduler.CosineAnnealingLR(\n", " opt, T_max=cfg.steps, eta_min=cfg.lr * 0.1) if cfg.lr_decay else None)\n", " alephs = model.aleph_layers()\n", " for a in alephs:\n", " a.emit_diversity = cfg.div_weight > 0\n", "\n", " snapshots: List[Tuple[int, Tensor]] = []\n", " if cfg.snapshot_codebook:\n", " snapshots.append((0, alephs[0].export_codebook()))\n", " _basin_state = {\"label\": statute(alephs[0].export_codebook())[\"statute\"]}\n", "\n", " print(f\"\\n=== AlephLM head={cfg.head} pi={cfg.pi_mode} \"\n", " f\"bank={cfg.bank_source if bank is not None else '-'} \"\n", " f\"params={n_params:,} ctx={cfg.seq_len} tri \"\n", " f\"eff.batch={cfg.batch_size * cfg.accum_steps} dev={dev} ===\")\n", " result: Dict = {\"head\": cfg.head, \"params\": n_params}\n", " t0 = time.time()\n", "\n", " def _save_ckpt():\n", " if not cfg.checkpoint_path:\n", " return\n", " tmp = cfg.checkpoint_path + \".tmp\"\n", " torch.save({\"model_state_dict\": model.state_dict(),\n", " \"config\": cfg.__dict__,\n", " \"bank\": model.bank.cpu() if model.bank is not None else None},\n", " tmp)\n", " os.replace(tmp, cfg.checkpoint_path)\n", " print(f\"[ckpt] -> {cfg.checkpoint_path}\")\n", " _hub_push(cfg.checkpoint_path)\n", "\n", " def _hub_push(path):\n", " if not cfg.hub_repo or not path:\n", " return\n", " try:\n", " from huggingface_hub import HfApi\n", " HfApi().upload_file(\n", " path_or_fileobj=path, repo_id=cfg.hub_repo,\n", " path_in_repo=f\"{cfg.hub_dir}/{os.path.basename(path)}\",\n", " commit_message=f\"autopush {os.path.basename(path)}\")\n", " print(f\"[hub] -> {cfg.hub_repo}/{cfg.hub_dir}/{os.path.basename(path)}\")\n", " except Exception as e:\n", " print(f\"[hub] push failed ({type(e).__name__}) — continuing; \"\n", " f\"local checkpoint is safe\")\n", "\n", " if cfg.init_from:\n", " d0 = torch.load(cfg.init_from, map_location=dev, weights_only=False)\n", " model.load_state_dict(d0[\"model_state_dict\"])\n", " print(f\"[warm-start] weights <- {cfg.init_from} (fresh optimizer)\")\n", "\n", " use_amp = cfg.amp and torch.device(dev).type == \"cuda\"\n", " if cfg.compile_backbone:\n", " model.backbone = torch.compile(model.backbone) # tensor-out only\n", "\n", " for step in range(1, cfg.steps + 1):\n", " opt.zero_grad(set_to_none=True)\n", " loss_sum, logs_acc = 0.0, {}\n", " for _ in range(cfg.accum_steps):\n", " if cfg.train_mode == \"stream\":\n", " ids, targets = stream.sample(\n", " cfg.batch_size, cfg.seq_len * cfg.segments, dev)\n", " states = None\n", " for si in range(cfg.segments):\n", " sl = slice(si * cfg.seq_len, (si + 1) * cfg.seq_len)\n", " with torch.autocast(device_type=\"cuda\",\n", " dtype=torch.bfloat16, enabled=use_amp):\n", " hseg, states = model.stream_backbone(\n", " ids[:, sl], states, si * cfg.seq_len)\n", " loss, logs = model.forward_loss(\n", " ids[:, sl], targets[:, sl], step=step, h=hseg)\n", " total = loss\n", " if cfg.div_weight > 0:\n", " total = total + cfg.div_weight * sum(\n", " a.diversity_loss() for a in alephs)\n", " (total / cfg.accum_steps / cfg.segments).backward()\n", " states = [tuple(t.detach() for t in st) for st in states]\n", " loss_sum += loss.item() / cfg.segments\n", " logs_acc = logs\n", " continue\n", " ids, targets = stream.sample(cfg.batch_size, cfg.seq_len, dev)\n", " with torch.autocast(device_type=\"cuda\",\n", " dtype=torch.bfloat16, enabled=use_amp):\n", " loss, logs = model.forward_loss(ids, targets, step=step)\n", " total = loss\n", " if cfg.div_weight > 0:\n", " total = total + cfg.div_weight * sum(\n", " a.diversity_loss() for a in alephs)\n", " (total / cfg.accum_steps).backward()\n", " loss_sum += loss.item()\n", " logs_acc = logs\n", " loss_avg = loss_sum / cfg.accum_steps\n", " gnorm = torch.nn.utils.clip_grad_norm_(\n", " model.parameters(), max(loss_avg, 1.0))\n", " opt.step()\n", " if sched is not None:\n", " sched.step()\n", " if cfg.ckpt_every and step % cfg.ckpt_every == 0:\n", " _save_ckpt()\n", "\n", " if step % cfg.log_every == 0 or step == cfg.steps:\n", " rate = step * cfg.batch_size * cfg.seq_len * cfg.accum_steps \\\n", " / (time.time() - t0)\n", " line = (f\" step {step:6d} loss {loss_avg:.4f} \"\n", " f\"bpb {logs_acc.get('bpb', logs_acc.get('bpb_inbank', float('nan'))):.3f} \"\n", " f\"|g| {gnorm:.2f} {rate/1e3:.1f}k tri/s\")\n", " if \"bpb_s\" in logs_acc:\n", " line += f\" bpbS {logs_acc['bpb_s']:.3f}\"\n", " if \"coverage\" in logs_acc:\n", " line += f\" cov {logs_acc['coverage']:.0%}\"\n", " if \"gate_acc\" in logs_acc:\n", " line += f\" gate {logs_acc['gate_acc']:.0%}\"\n", " if \"nn_exact\" in logs_acc:\n", " line += (f\" bpbE {logs_acc.get('bpb_eval', float('nan')):.3f}\"\n", " f\" nn {logs_acc['nn_exact']:.0%}\"\n", " f\" ang {logs_acc['ang_err_deg']:.1f}/\"\n", " f\"{logs_acc['budget_deg']:.1f}deg\"\n", " f\" inBudget {logs_acc['in_budget']:.0%}\")\n", " if \"bpb_bank_cond\" in logs_acc:\n", " line += (f\" inB {logs_acc['bpb_bank_cond']:.3f}\"\n", " f\" outB {logs_acc.get('bpb_byte_cond', float('nan')):.3f}\")\n", " if \"mode_spread_deg\" in logs_acc:\n", " line += (f\" spread {logs_acc['mode_spread_deg']:.0f}deg\"\n", " f\" mixH {logs_acc['mix_entropy']:.2f}\")\n", " if \"write_kl\" in logs_acc:\n", " line += f\" wKL {logs_acc['write_kl']:.3f}\"\n", " model.eval()\n", " with torch.no_grad():\n", " ids_p, _ = stream.sample(min(8, cfg.batch_size), cfg.seq_len, dev)\n", " st = alephs[0].address_stats(model.backbone(ids_p),\n", " max_rows=200_000)\n", " bg = model.branching_gauge(ids_p)\n", " model.train()\n", " line += (f\" ppl {st['perplexity']:.0f}/{st['max_perplexity']:.0f}\"\n", " f\" conf {bg['conf_mean']:.3f}\"\n", " f\"/{bg['kernel_invariant']:.3f}\"\n", " f\" branch {bg['branching_frac']:.0%}\")\n", " cb = alephs[0].export_codebook()\n", " st_now = statute(cb)\n", " sv = torch.linalg.svdvals(cb.float()) ** 2\n", " eff_rank = (sv.sum() ** 2 / (sv ** 2).sum()).item()\n", " line += (f\" dev {st_now['deviation']:+.3f}\"\n", " f\"/{st_now['statute'][:3]} rank {eff_rank:.2f}\")\n", " if st_now['statute'] != _basin_state[\"label\"]:\n", " print(f\"[basin] *** TRANSITION {_basin_state['label']} -> \"\n", " f\"{st_now['statute']} at step {step} \"\n", " f\"(dev {st_now['deviation']:+.4f}, eff.rank {eff_rank:.2f}) ***\")\n", " _basin_state[\"label\"] = st_now[\"statute\"]\n", " print(line)\n", " result.update(logs_acc)\n", " result.update({\"loss\": loss_avg, \"step\": step, **bg})\n", " if cfg.snapshot_codebook:\n", " snapshots.append((step, alephs[0].export_codebook()))\n", "\n", " if snapshots:\n", " traj = [(s, statute(cb)) for s, cb in snapshots]\n", " result[\"statute_trajectory\"] = traj\n", " torch.save({\"snapshots\": snapshots, \"statute_trajectory\": traj,\n", " \"config\": cfg.__dict__}, cfg.snapshot_path)\n", " _hub_push(cfg.snapshot_path)\n", " d0, d1 = traj[0][1][\"deviation\"], traj[-1][1][\"deviation\"]\n", " print(f\"\\n[basin] statute: dev {d0:+.4f} -> {d1:+.4f} \"\n", " f\"({traj[-1][1]['statute']}); snapshots -> {cfg.snapshot_path}\")\n", " _save_ckpt()\n", " return result\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke + activation\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " print(\"=\" * 70)\n", " print(\"AlephLM — smoke\")\n", " print(\"=\" * 70)\n", " rng = np.random.default_rng(0)\n", " words = [b\"the\", b\"aleph\", b\"predicts\", b\"its\", b\"own\", b\"future\",\n", " b\"through\", b\"a\", b\"codebook\"]\n", " path = \"/tmp/_alm_corpus.txt\"\n", " with open(path, \"wb\") as f:\n", " f.write(b\" \".join(words[i] for i in rng.integers(0, 9, 80000)))\n", "\n", " base = dict(corpus_id=path, max_corpus_bytes=None, steps=25, log_every=25,\n", " dim=96, n_layers=2, n_heads=4, K=16, seq_len=48, batch_size=8,\n", " bank_size=256, n_negatives=128, device=\"cpu\",\n", " checkpoint_path=None, snapshot_path=\"/tmp/_alm_snaps.pt\")\n", " prior_bpb = 8.0\n", " for head in (\"hybrid\", \"byte\", \"bank\", \"sampled\"):\n", " r = train_aleph_lm(AlephLMConfig(head=head, **base))\n", " bpb = r.get(\"bpb\", r.get(\"bpb_inbank\", float(\"nan\")))\n", " assert math.isfinite(r[\"loss\"]), head\n", " print(f\" ✓ head={head:8s} loss {r['loss']:.3f} bpb {bpb:.2f} \"\n", " f\"(uniform prior {prior_bpb:.1f})\")\n", " # pi ablation path + gradient to codebook through the PREDICT/CANDIDATE legs\n", " cfg = AlephLMConfig(head=\"hybrid\", pi_mode=\"address\", **base)\n", " stream = TrigramStream(path, max_corpus_bytes=None, seed=0)\n", " bank = build_corpus_bank(stream, cfg.bank_size)\n", " m = AlephLM(cfg, bank=bank)\n", " ids, tg = stream.sample(4, cfg.seq_len, \"cpu\")\n", " loss, _ = m.forward_loss(ids, tg)\n", " loss.backward()\n", " assert m.codebook.grad is not None and torch.isfinite(m.codebook.grad).all()\n", " print(f\" ✓ pi_mode='address' ablation runs; codebook grad |{m.codebook.grad.norm():.3f}|\")\n", " print(\"All smoke tests passed.\")\n", "\n", "\n", "def _max_spread_points(K: int, D: int, seed: int = 0,\n", " iters: int = 2000) -> torch.Tensor:\n", " \"\"\"Deterministic maximal-spread K points on S^(D-1): seeded init +\n", " sign-symmetric repulsion (minimize sum exp of |cos|), Adam, renormalized.\n", " Statute-by-construction codebook for freeze_codebook='spread'.\"\"\"\n", " g = torch.Generator().manual_seed(seed)\n", " x = torch.nn.Parameter(F.normalize(torch.randn(K, D, generator=g), dim=-1))\n", " o = torch.optim.Adam([x], lr=0.05)\n", " eye = torch.eye(K, dtype=torch.bool)\n", " # Annealed smooth-max packing: minimize logsumexp(beta*|cos|) with beta\n", " # rising, so optimization concentrates on the WORST pair (max-min angle\n", " # surrogate). Soft potentials provably prefer clustering here (orthoplex\n", " # of coincident points beats a spread under exp(4|c|) — measured 06-12).\n", " for beta in (8.0, 16.0, 32.0, 64.0):\n", " for _ in range(iters // 2):\n", " o.zero_grad()\n", " u = F.normalize(x, dim=-1)\n", " c = (u @ u.t()).masked_fill(eye, 0.0)\n", " loss = torch.logsumexp(beta * c.abs().flatten(), 0) / beta\n", " loss.backward()\n", " o.step()\n", " with torch.no_grad(): # saddle escape between rounds\n", " u = F.normalize(x, dim=-1)\n", " c = (u @ u.t()).masked_fill(eye, 0.0).abs()\n", " stuck = (c > 0.95).any(dim=1)\n", " if stuck.any():\n", " x[stuck] += 0.2 * torch.randn(int(stuck.sum()), D, generator=g)\n", " return F.normalize(x.detach(), dim=-1)\n", "\n", "\n", "def tier_a_config(bank_source: str, **overrides) -> AlephLMConfig:\n", " \"\"\"Tier A of the scaling plan (~25-30M params, one Blackwell, days):\n", " d=512 x 8 layers, J=16 pmix on a dense bank with sampled softmax,\n", " TBPTT streaming (4 x 1024 = effective 4096-trigram context at constant\n", " memory), clamped position (unbounded generation), bf16. Pure Adam,\n", " standing clip rule — nothing exotic enters with scale.\"\"\"\n", " base = dict(dim=512, n_layers=8, n_heads=8, K=64, d_point=48,\n", " head=\"hybrid\", bank_scorer=\"pmix\", n_pointers=16,\n", " bank_source=bank_source, bank_softmax=\"sampled\",\n", " n_bank_samples=8192, pos_mode=\"clamp\", train_mode=\"stream\",\n", " segments=4, seq_len=1024, batch_size=16, accum_steps=4,\n", " lr=5e-4, steps=50_000, amp=True, log_every=250)\n", " base.update(overrides)\n", " return AlephLMConfig(**base)\n", "\n", "\n", "def _running_in_notebook() -> bool:\n", " \"\"\"Pasted Colab/Jupyter cells run with __name__ == '__main__'; this keeps\n", " the demo/CLI below inert there so pasting never auto-fires it.\"\"\"\n", " try:\n", " from IPython import get_ipython\n", " return get_ipython() is not None\n", " except Exception:\n", " return False\n", "\n", "\n", "if __name__ == \"__main__\" and not _running_in_notebook():\n", " import argparse\n", " ap = argparse.ArgumentParser(description=\"AlephLM — prediction through the codebook\")\n", " ap.add_argument(\"--smoke-only\", action=\"store_true\")\n", " ap.add_argument(\"--head\", default=\"hybrid\",\n", " choices=[\"hybrid\", \"byte\", \"bank\", \"sampled\", \"pointer\"])\n", " ap.add_argument(\"--bank\", default=\"corpus\", choices=[\"corpus\", \"wordnet\"])\n", " ap.add_argument(\"--pi\", default=\"free\", choices=[\"free\", \"address\"])\n", " ap.add_argument(\"--scorer\", default=\"kernel\", choices=[\"kernel\", \"pmix\"])\n", " ap.add_argument(\"--pointers\", type=int, default=4)\n", " ap.add_argument(\"--steps\", type=int, default=10_000)\n", " ap.add_argument(\"--corpus-mb\", type=int, default=100)\n", " ap.add_argument(\"--device\",\n", " default=\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " args, _unknown = ap.parse_known_args()\n", " if args.smoke_only:\n", " _smoke()\n", " else:\n", " cfg = AlephLMConfig(head=args.head, bank_source=args.bank,\n", " pi_mode=args.pi, steps=args.steps,\n", " bank_scorer=args.scorer, n_pointers=args.pointers,\n", " max_corpus_bytes=args.corpus_mb * 1_000_000,\n", " device=args.device)\n", " train_aleph_lm(cfg)" ], "metadata": { "id": "C8Fi24hh5erX" }, "execution_count": 7, "outputs": [] }, { "cell_type": "markdown", "source": [ "# run lm adapter" ], "metadata": { "id": "LDoo8h1L7A2K" } }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_lm_adapter.py — exp_011 Phase 3 (Tier-L)\n", "# Composed micro-aleph addresses conditioning the byte-trigram LM.\n", "#\n", "# Injection seam (source-grounded, aleph_lm.py L474/L610): every head reads\n", "# h = backbone(ids), so conditioning h routes prediction THROUGH the composed\n", "# structure — the employment law, satisfied with zero head surgery:\n", "#\n", "# h = AlephLM.backbone(ids) # (B, S, d)\n", "# h' = h + alpha * W_out( ACD(h) ) # alpha init 0.1 (the\n", "# # simplex-injection precedent)\n", "#\n", "# Tier-L metrics map (documented, units differ from Tier-P):\n", "# ce_bits = bpb (bits per byte, the LM's delivered channel)\n", "# delivered_bits = H0 - bpb (bits ABOVE the measured unigram floor;\n", "# H0 = bias-only probe entropy, so held and delivered\n", "# share one baseline and one scale)\n", "# cum/marginal = staged probes predicting the NEXT BYTE (256-way)\n", "# from cached token addresses (same estimator, same\n", "# class-scaled budget)\n", "#\n", "# Paste order: acd_structures -> acd_probe -> aleph_lm (cells 1-4 of the\n", "# -lm repo, or `from aleph_lm import ...`) -> acd_lm_adapter -> acd_forge.\n", "# ============================================================\n", "from __future__ import annotations\n", "import math, time\n", "from dataclasses import dataclass\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure\n", " from acd_probe import _probe_ce_bits, composition_gauges\n", " from aleph_lm import AlephLM, AlephLMConfig\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver. Pasted Colab cells share ONE namespace and are\n", " not importable modules — so resolve names from globals() first (paste\n", " mode), then fall back to a real import (script/module mode).\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig — the canonical Tier-L recipe (exp_010's 6.75M, sweep-proven)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def tier_l_overrides(**extra) -> Dict:\n", " \"\"\"dim=384 x 4 layers, the exp_010 small backbone. Colab default.\n", " head='byte' keeps the adapter bank-free; hybrid/apmix arms pass\n", " bank_source explicitly (exp_010 convention).\"\"\"\n", " base = dict(dim=384, n_layers=4, n_heads=6, K=64, seq_len=256,\n", " head=\"byte\", batch_size=32, lr=5e-4, amp=False)\n", " base.update(extra)\n", " return base\n", "\n", "\n", "def smoke_overrides(**extra) -> Dict:\n", " base = dict(dim=64, n_layers=2, n_heads=2, K=8, seq_len=64,\n", " head=\"byte\", batch_size=8, lr=3e-3, amp=False)\n", " base.update(extra)\n", " return base\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# ACDConditioner — token-wise composed address, alpha-gated residual\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class ACDConditioner(nn.Module):\n", " \"\"\"h (B,S,d) -> h + alpha * W_out(ACD(h)). Caches the per-token stage\n", " addresses (subsampled) for the gauge battery. No norm/dropout on the\n", " geometric path (statute).\"\"\"\n", "\n", " def __init__(self, d_model: int, acd_cfg: \"ACDConfig\",\n", " alpha_init: float = 0.1, cache_positions: int = 2048):\n", " super().__init__()\n", " acd_cfg.d_in = d_model\n", " self.acd = ACDStructure(acd_cfg)\n", " self.out = nn.Linear(self.acd.cfg.feature_dim, d_model, bias=False)\n", " torch.nn.init.orthogonal_(self.out.weight)\n", " self.alpha = nn.Parameter(torch.tensor(float(alpha_init)))\n", " self.cache_positions = cache_positions\n", " self.cached_addresses: Optional[Tensor] = None # (N, m, 2K_max)\n", " self.cached_flat_idx: Optional[Tensor] = None # (N,) into B*S\n", "\n", " def forward(self, h: Tensor) -> Tensor:\n", " B, S, d = h.shape\n", " flat = h.reshape(B * S, d)\n", " feats, addrs = self.acd(flat) # (BS,F), (BS,m,2K)\n", " if not self.training:\n", " n = min(self.cache_positions, B * S)\n", " idx = torch.randperm(B * S, device=h.device)[:n]\n", " self.cached_addresses = addrs[idx].detach()\n", " self.cached_flat_idx = idx\n", " return h + self.alpha * self.out(feats).reshape(B, S, d)\n", "\n", "\n", "_ACD_ALEPH_LM_CLS = None\n", "\n", "def acd_aleph_lm_cls():\n", " \"\"\"Lazy subclass factory: pasted cells cannot subclass a base that\n", " hasn't been pasted yet, so ACDAlephLM materializes at FIRST USE,\n", " resolving AlephLM through the shared namespace (or import). Cached;\n", " also published to globals() as ACDAlephLM once built.\"\"\"\n", " global _ACD_ALEPH_LM_CLS\n", " if _ACD_ALEPH_LM_CLS is not None:\n", " return _ACD_ALEPH_LM_CLS\n", " _AlephLM = _ns(\"AlephLM\", \"aleph_lm\")\n", "\n", " class ACDAlephLM(_AlephLM):\n", " \"\"\"AlephLM whose backbone output is ACD-conditioned. Every head —\n", " byte, kernel, pmix, apmix — reads through the composed structure.\"\"\"\n", "\n", " def __init__(self, cfg, acd_cfg: \"ACDConfig\",\n", " alpha_init: float = 0.1):\n", " super().__init__(cfg)\n", " self.conditioner = ACDConditioner(cfg.dim, acd_cfg, alpha_init)\n", "\n", " def backbone(self, ids: Tensor) -> Tensor:\n", " return self.conditioner(super().backbone(ids))\n", "\n", " def acd_param_groups(self, lr: float, geom_lr_mult: float = 0.1):\n", " \"\"\"Optimizer groups per the standing Adam-alignment rule:\n", " codebooks on a slow lane, everything else at base lr.\"\"\"\n", " geom_ids, geom = set(), []\n", " for n, p in self.named_parameters():\n", " if \"codebook\" in n or \"branch_books\" in n:\n", " geom_ids.add(id(p)); geom.append(p)\n", " rest = [p for p in self.parameters() if id(p) not in geom_ids]\n", " return [dict(params=rest, lr=lr),\n", " dict(params=geom, lr=lr * geom_lr_mult, weight_decay=0.0)]\n", "\n", " _ACD_ALEPH_LM_CLS = ACDAlephLM\n", " globals()[\"ACDAlephLM\"] = ACDAlephLM\n", " return ACDAlephLM\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Synthetic stream — order-2 Markov bytes (sandbox smoke; Colab uses\n", "# TrigramStream on WikiText, same API surface)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class MarkovByteStream:\n", " \"\"\"Learnable-structure byte source: 48-symbol alphabet, sparse order-2\n", " transition table. sample(B, S, device) -> ids, targets: (B, S, 3) longs\n", " (next-trigram prediction framing, mirrors TrigramStream).\"\"\"\n", "\n", " def __init__(self, seed: int = 0, n_sym: int = 48, branch: int = 4,\n", " length: int = 200_000):\n", " g = torch.Generator().manual_seed(seed)\n", " nxt = torch.randint(0, n_sym, (n_sym, n_sym, branch), generator=g)\n", " buf = torch.empty(length, dtype=torch.uint8)\n", " a = b = 0\n", " for i in range(length):\n", " c = nxt[a, b, torch.randint(0, branch, (1,), generator=g)].item()\n", " buf[i] = 32 + c # printable-ish bytes\n", " a, b = b, c\n", " self.stream = buf\n", " self._g = g\n", "\n", " def sample(self, batch: int, seq_len: int, device=\"cpu\"):\n", " need = 3 * (seq_len + 1)\n", " starts = torch.randint(0, len(self.stream) - need - 3,\n", " (batch,), generator=self._g)\n", " starts = (starts // 3) * 3\n", " rows = torch.stack([self.stream[s:s + need] for s in starts]).long()\n", " tri = rows.view(batch, seq_len + 1, 3)\n", " return tri[:, :-1].to(device), tri[:, 1:].to(device)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier-L gauges — next-byte staged probes on cached token addresses\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@torch.no_grad()\n", "def _collect_lm(model: \"ACDAlephLM\", stream, seq_len: int,\n", " batches: int, batch: int, device):\n", " model.eval()\n", " A, Y = [], []\n", " for _ in range(batches):\n", " ids, tg = stream.sample(batch, seq_len, device)\n", " model.backbone(ids) # fills the cache\n", " addrs = model.conditioner.cached_addresses # (N, m, 2K)\n", " idx = model.conditioner.cached_flat_idx\n", " y = tg[..., 0].reshape(-1)[idx] # next byte 0\n", " A.append(addrs); Y.append(y)\n", " return torch.cat(A), torch.cat(Y)\n", "\n", "\n", "def lm_marginal_bits(model, stream, seq_len: int, device,\n", " batches: int = 4, batch: int = 16,\n", " probe_steps: int = 640, seed: int = 0) -> Dict:\n", " \"\"\"H0 REBASING: the empirical byte distribution is far from uniform\n", " (WikiText unigram entropy ~4.4 bits, not 8), so a probe on ANY input\n", " \"recovers\" ~3.6 bits of pure prior. H0 = bias-only probe (constant\n", " input) measures that floor; marginals and the curve are reported as\n", " BITS ABOVE UNIGRAM. Stage-1 marginal = H0 - H(Y|a_1).\"\"\"\n", " addrs, y = _collect_lm(model, stream, seq_len, batches, batch, device)\n", " N, m, W = addrs.shape\n", " ones = torch.ones(N, 1, device=addrs.device)\n", " H0 = _probe_ce_bits(ones, y, 256, steps=probe_steps, seed=seed - 1)\n", " H0 = min(H0, 8.0)\n", " H_prev = H0\n", " curve, marg, acc = [], [], 0.0\n", " for t in range(m):\n", " prefix = addrs[:, : t + 1].reshape(N, -1)\n", " out = _probe_ce_bits(prefix, y, 256, steps=probe_steps,\n", " seed=seed + t, return_acc=(t == m - 1))\n", " H_t, acc = out if t == m - 1 else (out, acc)\n", " H_t = min(H_t, H_prev)\n", " marg.append(H_prev - H_t)\n", " curve.append(H0 - H_t)\n", " H_prev = H_t\n", " return {\"marginal_bits\": marg, \"cumulative_bits\": curve,\n", " \"H0\": H0, \"probe_acc\": acc, \"addresses\": addrs}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# run_arm_L — one arm, one rung (forge row schema)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_arm_L(spec, steps: int, stream, lm_over: Dict,\n", " device: str, probe_steps: int = 640,\n", " eval_every: int = 100) -> Dict[str, object]:\n", " import json as _json\n", " torch.manual_seed(spec.seed)\n", " _Cfg = _ns(\"AlephLMConfig\", \"aleph_lm\")\n", " cfg = _Cfg(**lm_over)\n", " acd = spec.to_acd(d_in=cfg.dim, feature_dim=min(128, 2 * cfg.dim))\n", " model = acd_aleph_lm_cls()(cfg, acd).to(device)\n", " opt = torch.optim.Adam(model.acd_param_groups(lm_over.get(\"lr\", 5e-4)))\n", " t0, killed = time.time(), None\n", " model.train()\n", " for step in range(1, steps + 1):\n", " ids, tg = stream.sample(cfg.batch_size, cfg.seq_len, device)\n", " loss, _aux = model.forward_loss(ids, tg)\n", " opt.zero_grad()\n", " loss.backward()\n", " gn = torch.nn.utils.clip_grad_norm_(model.parameters(),\n", " max(loss.item(), 1.0))\n", " opt.step()\n", " if step % eval_every == 0 and not math.isfinite(loss.item()):\n", " killed = \"NaN/inf loss\"\n", " break\n", " # eval bpb on held batches\n", " model.eval()\n", " with torch.no_grad():\n", " tot, n = 0.0, 0\n", " for _ in range(4):\n", " ids, tg = stream.sample(cfg.batch_size, cfg.seq_len, device)\n", " l, _ = model.forward_loss(ids, tg)\n", " tot += l.item(); n += 1\n", " bpb = (tot / n) / math.log(2) / 3.0 # nats/trigram -> bits/byte\n", " mb = lm_marginal_bits(model, stream, cfg.seq_len, device,\n", " probe_steps=probe_steps, seed=spec.seed)\n", " H0 = mb[\"H0\"]\n", " cg = composition_gauges(model.conditioner.acd,\n", " torch.zeros(1, cfg.dim, device=device)) \\\n", " if False else _addr_gauges(mb[\"addresses\"])\n", " st = model.conditioner.acd.codebook_stats()\n", " return dict(arm_id=spec.arm_id(), op=spec.op, m=spec.m, K=spec.K,\n", " d_addr=spec.d_addr, freeze=spec.freeze, seed=spec.seed,\n", " tree_hard=spec.tree_hard, steps=steps,\n", " params=sum(p.numel() for p in model.parameters()),\n", " acc=round(mb[\"probe_acc\"], 4), ce_bits=round(bpb, 4),\n", " delivered_bits=round(H0 - bpb, 4), # bits above unigram\n", " cum_bits=round(mb[\"cumulative_bits\"][-1], 4),\n", " marginal_bits=_json.dumps(\n", " [round(v, 4) for v in mb[\"marginal_bits\"]]),\n", " redundancy=cg[\"redundancy\"],\n", " cancellation=cg[\"cancellation\"], stage_snr=0.0,\n", " dev_mean=round(sum(s[\"deviation\"] for s in st) / len(st), 4),\n", " rank_mean=round(sum(s[\"eff_rank\"] for s in st) / len(st), 3),\n", " killed=killed or \"\", wall_s=round(time.time() - t0, 1))\n", "\n", "\n", "@torch.no_grad()\n", "def _addr_gauges(addrs: Tensor) -> Dict[str, float]:\n", " \"\"\"redundancy/cancellation on cached (N, m, 2K) token addresses —\n", " same definitions as acd_probe.composition_gauges.\"\"\"\n", " N, m, W = addrs.shape\n", " K = W // 2\n", " if m == 1:\n", " return {\"redundancy\": 0.0, \"cancellation\": 0.0}\n", " flat = F.normalize(addrs, dim=-1)\n", " cos = torch.einsum(\"bmw,bnw->bmn\", flat, flat)\n", " iu = torch.triu_indices(m, m, offset=1)\n", " red = cos[:, iu[0], iu[1]].mean().item()\n", " net = addrs[..., :K] - addrs[..., K:]\n", " agree = (net.sign().sum(dim=1).abs() == m).float()\n", " canc = (((1 - agree) * net.abs().sum(1)).sum(-1)\n", " / net.abs().sum(dim=(1, 2)).clamp_min(1e-9)).mean().item()\n", " return {\"redundancy\": round(red, 4), \"cancellation\": round(canc, 4)}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke — synthetic stream, tiny LM, prod_m4 conditioner\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " try:\n", " ArmSpec = _ns(\"ArmSpec\", \"acd_forge\")\n", " except Exception:\n", " # forge not pasted yet (it comes AFTER the adapter): a minimal shim\n", " # covering exactly the surface run_arm_L touches. Real runs always\n", " # use the forge grammar; the shim exists only for this smoke.\n", " from dataclasses import dataclass as _dc\n", " @_dc\n", " class ArmSpec:\n", " op: str\n", " m: int\n", " K: int = 0\n", " d_addr: int = 4\n", " freeze: str = \"free\"\n", " seed: int = 1234\n", " tree_hard: bool = False\n", " def resolve(self):\n", " if self.K == 0:\n", " self.K = max(2, 256 // (max(self.m, 1) * self.d_addr))\n", " if self.op == \"single\":\n", " self.m = 1\n", " return self\n", " def arm_id(self):\n", " return f\"smoke_{self.op}_m{self.m}_K{self.K}_s{self.seed}\"\n", " def to_acd(self, d_in, feature_dim=64):\n", " return ACDConfig(op=self.op, d_in=d_in, m=self.m, K=self.K,\n", " d_addr=self.d_addr, freeze=self.freeze,\n", " tree_hard=self.tree_hard, seed=self.seed,\n", " feature_dim=feature_dim)\n", " print(\" - ArmSpec shim in use (real grammar arrives with acd_forge)\")\n", " dev = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " stream = MarkovByteStream(seed=3)\n", " ids, tg = stream.sample(4, 32, dev)\n", " assert ids.shape == (4, 32, 3) and tg.shape == (4, 32, 3)\n", " print(f\" ✓ MarkovByteStream: trigram framing {tuple(ids.shape)}\")\n", "\n", " try:\n", " _ns(\"AlephLM\", \"aleph_lm\")\n", " except Exception:\n", " print(\" - aleph-lm cells not in namespace yet: arm smoke DEFERRED \"\n", " \"(paste cells 1-4, then re-run _smoke() — or it validates on \"\n", " \"the first phase3 arm)\")\n", " print(\"acd_lm_adapter smoke: PARTIAL (stream green, arms deferred)\")\n", " return\n", " over = smoke_overrides()\n", " spec = ArmSpec(op=\"prod\", m=4, K=8, seed=1234).resolve()\n", " row = run_arm_L(spec, steps=40, stream=stream, lm_over=over,\n", " device=dev, probe_steps=200, eval_every=20)\n", " assert math.isfinite(row[\"ce_bits\"]) and row[\"ce_bits\"] < 8.0\n", " assert isinstance(row[\"delivered_bits\"], float)\n", " assert -1.0 < row[\"delivered_bits\"] < 8.0 # rebased scale sanity\n", " assert 0.0 <= row[\"acc\"] <= 1.0 # probe top-1\n", " import json as _json\n", " marg = _json.loads(row[\"marginal_bits\"])\n", " assert len(marg) == 4 and all(v >= 0 for v in marg)\n", " print(f\" ✓ run_arm_L: bpb {row['ce_bits']:.3f} delivered \"\n", " f\"{row['delivered_bits']:.3f} held {row['cum_bits']:.3f} \"\n", " f\"{marg} red {row['redundancy']:.2f} \"\n", " f\"dev {row['dev_mean']:+.3f} rank {row['rank_mean']:.2f}\")\n", "\n", " # single-arm control path (m=1 gauges degenerate cleanly)\n", " s1 = ArmSpec(op=\"single\", m=1, K=32, seed=1234).resolve()\n", " r1 = run_arm_L(s1, steps=20, stream=stream, lm_over=over,\n", " device=dev, probe_steps=150, eval_every=20)\n", " assert r1[\"redundancy\"] == 0.0\n", " print(f\" ✓ single control: bpb {r1['ce_bits']:.3f}\")\n", "\n", " # alpha gate is learnable and moved\n", " print(\"acd_lm_adapter smoke: ALL GREEN\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Yr0LxOuf41hC", "outputId": "3c6052f7-3581-42d5-8fe7-8f99dd88e6fe" }, "execution_count": 10, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ MarkovByteStream: trigram framing (4, 32, 3)\n", " ✓ run_arm_L: bpb 5.598 delivered 0.039 held 0.036 [0.0193, 0.0, 0.0004, 0.0161] red 0.16 dev -0.010 rank 3.57\n", " ✓ single control: bpb 5.636\n", "acd_lm_adapter smoke: ALL GREEN\n" ] } ] }, { "cell_type": "code", "source": [ "phase3_screen()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "83d1d83f941942e5877d10dda18831ef", "5484cc8c987c4b90972056a2a1ad647e", "eeaa111590424f478fb4d7372d0949d5", "400aabdf6f71482caf36df6cb7650c5e", "b912b58d41b34c35b545119b1124e1a9", "371c6596a5ba426c944ee344bc82e15a", "5cd57eea1d3649d9a929a233ef9cf1b3", "d44f66cc06f34380979de8619df5052d", "bb2b5d1d2f3945e1b03a8845df7600d9", "4445cf712c5648a291608f0d28b2c546", "4053dbd62e1a4997bb211d8a061ffabf", "50e150adf12c4eac83c5bbbcba495436", "f82a7651dcfd4912bc86603194e194cf", "f38eb0917dfe4fc2bc26a687b4d79c21", "2708c3a73e7145c8955146cd5547a91e", "7121279632654d52adcc7eff3aace621", "dffad1378e614dc2921cb50326fe3170", "887bf4165de24db8aecf648ff3ff0fb5", "60712f450afb443683564734a6013d0b", "8fe736e3baa2475189bc9f9a79dcd5d0", "798d0ff4fcb74bd6842425a14d865bb2", "fc31527a15c049ea968b7a0727329bcb", "37238e991469495caee69628b6ea1956", "f91c35e715c7440d9db9cf88ff834a87", "dcf30fd8ca1743c799fedde260d4770e", "1ccc2d2dd89845d4a9f94da3f27e85cd", "9ee3cbae883a4743b6d1610b4d0a53de", "4c6ad259d67049a085d256445911e6cf", "89b2237ac6fe4e44bd48c198c9d53c48", "7a6e63d395e94e5a99edd973db882b5e", "ea161dc0e5ae400f85c28dc8f5fd276f", "76b0687fe246471d9008590a9bad7153", "5575dd05ed4e4c2daeb1d0d96742a499", "12dd0d2094df41389d65356f0237e67a", "5efd12e892ae47debb1fbf6ee81c0251", "bf95ac59c2cc489c8f58afeef48198ca", "7226093a972c43129145f336ac7c87d9", "269f7e0e536e4d73917a47aa8af29400", "4500d3252dd140cb915667797ab1e440", "fffb559b9ce24b97bb4002c381ba3f51", "e3b4475f645d4ca189ad0655cbaac298", "e84425f852614e4993dcec4f74a7ea46", "7146d0488639440b80ac105efb0bcf11", "0609e309f35c459cb46f432f39ce5b39", "9893c161f37f4409b9267abe5c80afce", "111f5d3ceca84420a33e0d6811d61be0", "4992d11daab6406e8ee6aa36014e93c5", "9e520a8a2af64f2e8efc02d0507eca04", "c479d075853b4463bc3dfbe85b432ab6", "4d0a0a58d7f5437fb044d45cd7a55d23", "86e6fdce583242f1a24f386cfc276ff1", "caed3676a4744808af3d3b899206c23c", "b3a8fab92e41487d980d899ab73515d8", "8c460ad9d9754f0eac89d4c98bd03350", "662a8c74eccc4780801b356887b19ad3", "521bbe2d2c5940e8839ea7a649f23164", "047dc9fe110c4bc18641fc6366b2e0de", "ebf222094ac14b3392c7b5dfe199ddae", "9d55ee1c086f426eb573b0eb0886bcb5", "2ef6b544d49c482a85cb59fee51aaa41", "03b16f9eb06d44e898b84bce8da58ced", "1546b0d321eb416183f14b94785b48a8", "05fc7e21c5cb42c7b7738e41bb822612", "8c2dae19f12e4e99908980e41ddaadc2", "f27db7cc8e3145708c84d0527de74c60", "9e3ecb45249b4367a98c60bad71d2873", "83a79438586c45caafca24b7f3dc2449", "a5d6a4a8ee944d2c83df5dd2a89c33d9", "6c97d3e49fab4f9f8015537705e92ddf", "0ffef704b9a64e1189c44145ddc6eeb7", "cae54e625ff241f687fba6d3cb7ab241", "e9d638d1dc69450e97e8d875694acfc1", "400cd959440f496fa2b3295d6ddfed75", "d524ae6998a04a9a9adaede5917d1b2f", "8a2a22afa3184a88b3f6ee57bd0811d1", "d59c2f1df60543b3a081bc8bfd6f1f2f", "8cbaa69fb86d489ba01f305993160710", "55c498bc304947c2805cf7db7e6cfd95", "5f419d0cc5aa49caae4aec2e3257a5db", "e0ab10aa67114492a2f77587b1570783", "ebf6e51dc43b439a8842b9957d7db8eb", "8d4b5312ea304c0ea732ed1b096610da", "ea40ffbd34c743369f31595038d22c98", "5c1698382dee4cb79498be10ca625193", "87c0bd1330e84a8d88d0a2c03619e9fa", "e3e1258c184a4cdb9f17cae93458e070", "8155a51d8a5f4b7b9872bf5a0de61407", "81e301c3f8804a31b880c8ad8a6ddc55" ] }, "id": "efSVB0US9glv", "outputId": "4c5f9b19-a4b0-482f-94ad-17d9ddb70c33" }, "execution_count": 11, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[forge] 12 Tier-L arms queued (sum+single protected)\n", "[TrigramStream] loading HF corpus wikitext-103-raw-v1 ...\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "README.md: 0%| | 0.00/10.5k [00:00 AbstractPhil/geolip-aleph-differentiation ✓\n", "\n", "[rung 1] tier=L steps=5000 arms=10 (cached 0, fresh 10)\n", " [1/10] res_m16_K4_free_s1234_52ba39 held 1.10 ce 2.800 dlv 1.80 [0.173, 0.0477, 0.0386, 0.3893, 0.0332, 0.0743, 0.1745, 0.019, 0.0, 0.0482, 0.0232, 0.0567, 0.0, 0.0057, 0.0183, 0.0] acc 0.337 red 0.14\n", " [2/10] single_m1_K64_free_s1234_194318 held 0.08 ce 2.813 dlv 1.77 [0.0789] acc 0.195 red 0.00\n", " [3/10] sum_m16_K4_free_s5678_a631de held 1.11 ce 2.803 dlv 1.79 [0.1179, 0.4659, 0.1057, 0.0702, 0.0594, 0.0243, 0.0, 0.1206, 0.0, 0.0087, 0.0766, 0.0376, 0.0, 0.0, 0.0214, 0.0] acc 0.325 red 0.33\n", " [4/10] sum_m16_K4_free_s1234_85541a held 1.17 ce 2.822 dlv 1.79 [0.2555, 0.2261, 0.0993, 0.1091, 0.0491, 0.074, 0.0919, 0.0353, 0.0872, 0.0, 0.0028, 0.0282, 0.0326, 0.0258, 0.0261, 0.0235] acc 0.352 red 0.23\n", " [5/10] res_m8_K8_free_s1234_9db65b held 1.05 ce 2.834 dlv 1.71 [0.1654, 0.1425, 0.0577, 0.5625, 0.0606, 0.0025, 0.0327, 0.0299] acc 0.333 red 0.14\n", " [6/10] res_m16_K4_free_s5678_d700ae held 1.30 ce 2.827 dlv 1.80 [0.4678, 0.2041, 0.0737, 0.3107, 0.0024, 0.0559, 0.0, 0.0757, 0.0, 0.0, 0.0638, 0.036, 0.0, 0.0017, 0.0, 0.0055] acc 0.361 red 0.20\n", " [7/10] res_m8_K8_free_s5678_0c1ab5 held 1.19 ce 2.818 dlv 1.77 [0.171, 0.3499, 0.1002, 0.0986, 0.0, 0.2152, 0.2516, 0.0] acc 0.349 red 0.10\n", " [8/10] sum_m8_K8_free_s1234_7106f9 held 0.85 ce 2.793 dlv 1.81 [0.2242, 0.1211, 0.0114, 0.3093, 0.0603, 0.0973, 0.0248, 0.0016] acc 0.296 red 0.13\n", " [9/10] sum_m8_K8_free_s5678_ccb02a held 0.76 ce 2.829 dlv 1.78 [0.0698, 0.46, 0.0, 0.0248, 0.1357, 0.0, 0.0299, 0.0383] acc 0.268 red 0.14\n", " [10/10] single_m1_K64_free_s5678_800880 held 0.10 ce 2.794 dlv 1.80 [0.1033] acc 0.202 red 0.00\n", "[push] exp011L/ -> AbstractPhil/geolip-aleph-differentiation ✓\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "[{'arm_id': 'prod_m8_K8_free_s1234_e1d889',\n", " 'op': 'prod',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6968899,\n", " 'acc': 0.272,\n", " 'ce_bits': 3.2881,\n", " 'delivered_bits': 1.3518,\n", " 'cum_bits': 0.7158,\n", " 'marginal_bits': '[0.434, 0.0796, 0.0047, 0.0688, 0.0, 0.1287, 0.0, 0.0]',\n", " 'redundancy': 0.0415,\n", " 'cancellation': 0.9974,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0167,\n", " 'rank_mean': 3.623,\n", " 'killed': '',\n", " 'wall_s': 29.3,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'prod_m8_K8_free_s5678_51ec06',\n", " 'op': 'prod',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6968899,\n", " 'acc': 0.2446,\n", " 'ce_bits': 3.2508,\n", " 'delivered_bits': 1.2633,\n", " 'cum_bits': 0.4669,\n", " 'marginal_bits': '[0.04, 0.0425, 0.0453, 0.0672, 0.0418, 0.016, 0.1094, 0.1046]',\n", " 'redundancy': 0.1066,\n", " 'cancellation': 0.9998,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0013,\n", " 'rank_mean': 3.603,\n", " 'killed': '',\n", " 'wall_s': 28.7,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m8_K8_free_s1234_9db65b',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6881347,\n", " 'acc': 0.2996,\n", " 'ce_bits': 3.2418,\n", " 'delivered_bits': 1.385,\n", " 'cum_bits': 0.9386,\n", " 'marginal_bits': '[0.2631, 0.0909, 0.1848, 0.2222, 0.0961, 0.0148, 0.0, 0.0667]',\n", " 'redundancy': 0.0927,\n", " 'cancellation': 0.9976,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0166,\n", " 'rank_mean': 3.622,\n", " 'killed': '',\n", " 'wall_s': 28.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/12 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_free_s5678_0c1ab5',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6881347,\n", " 'acc': 0.2739,\n", " 'ce_bits': 3.2357,\n", " 'delivered_bits': 1.3418,\n", " 'cum_bits': 0.7435,\n", " 'marginal_bits': '[0.1531, 0.2389, 0.0717, 0.0577, 0.0, 0.1328, 0.0893, 0.0]',\n", " 'redundancy': 0.1467,\n", " 'cancellation': 0.9797,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0011,\n", " 'rank_mean': 3.603,\n", " 'killed': '',\n", " 'wall_s': 29.0,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/12 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s1234_7106f9',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6832203,\n", " 'acc': 0.2915,\n", " 'ce_bits': 3.2658,\n", " 'delivered_bits': 1.3212,\n", " 'cum_bits': 0.7638,\n", " 'marginal_bits': '[0.1133, 0.0459, 0.0419, 0.2722, 0.0396, 0.0672, 0.0146, 0.1691]',\n", " 'redundancy': 0.2459,\n", " 'cancellation': 0.994,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0165,\n", " 'rank_mean': 3.623,\n", " 'killed': '',\n", " 'wall_s': 29.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/12 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s5678_ccb02a',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6832203,\n", " 'acc': 0.2866,\n", " 'ce_bits': 3.2555,\n", " 'delivered_bits': 1.3174,\n", " 'cum_bits': 0.674,\n", " 'marginal_bits': '[0.1986, 0.2021, 0.0, 0.0081, 0.1654, 0.0947, 0.0052, 0.0]',\n", " 'redundancy': 0.1866,\n", " 'cancellation': 0.9998,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0015,\n", " 'rank_mean': 3.603,\n", " 'killed': '',\n", " 'wall_s': 29.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6820547,\n", " 'acc': 0.1987,\n", " 'ce_bits': 3.2454,\n", " 'delivered_bits': 1.3911,\n", " 'cum_bits': 0.1512,\n", " 'marginal_bits': '[0.1512]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0067,\n", " 'rank_mean': 3.931,\n", " 'killed': '',\n", " 'wall_s': 22.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_free_s5678_800880',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6820547,\n", " 'acc': 0.1797,\n", " 'ce_bits': 3.2549,\n", " 'delivered_bits': 1.3578,\n", " 'cum_bits': 0.0198,\n", " 'marginal_bits': '[0.0198]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0063,\n", " 'rank_mean': 3.977,\n", " 'killed': '',\n", " 'wall_s': 22.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m16_K4_free_s1234_52ba39',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6894659,\n", " 'acc': 0.3196,\n", " 'ce_bits': 3.2401,\n", " 'delivered_bits': 1.3461,\n", " 'cum_bits': 0.9495,\n", " 'marginal_bits': '[0.1174, 0.0341, 0.0426, 0.3941, 0.0648, 0.1207, 0.024, 0.0057, 0.1193, 0.0042, 0.0, 0.0004, 0.0017, 0.0207, 0.0, 0.0]',\n", " 'redundancy': 0.1445,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0058,\n", " 'rank_mean': 2.94,\n", " 'killed': '',\n", " 'wall_s': 36.6,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/12 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_free_s5678_d700ae',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6894659,\n", " 'acc': 0.3274,\n", " 'ce_bits': 3.2635,\n", " 'delivered_bits': 1.3699,\n", " 'cum_bits': 0.9829,\n", " 'marginal_bits': '[0.1428, 0.1854, 0.0015, 0.4811, 0.0, 0.0, 0.0, 0.0402, 0.0363, 0.0, 0.0471, 0.0081, 0.0371, 0.0033, 0.0, 0.0]',\n", " 'redundancy': 0.1693,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0088,\n", " 'rank_mean': 3.03,\n", " 'killed': '',\n", " 'wall_s': 36.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/12 by cum_bits'},\n", " {'arm_id': 'sum_m16_K4_free_s1234_85541a',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6845523,\n", " 'acc': 0.2676,\n", " 'ce_bits': 3.2557,\n", " 'delivered_bits': 1.3793,\n", " 'cum_bits': 0.8173,\n", " 'marginal_bits': '[0.2164, 0.2521, 0.0486, 0.0, 0.0351, 0.0875, 0.0014, 0.0286, 0.0244, 0.0305, 0.0, 0.0, 0.0457, 0.047, 0.0, 0.0]',\n", " 'redundancy': 0.2419,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0048,\n", " 'rank_mean': 2.949,\n", " 'killed': '',\n", " 'wall_s': 37.0,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/12 by cum_bits'},\n", " {'arm_id': 'sum_m16_K4_free_s5678_a631de',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6845523,\n", " 'acc': 0.238,\n", " 'ce_bits': 3.2393,\n", " 'delivered_bits': 1.3989,\n", " 'cum_bits': 0.6438,\n", " 'marginal_bits': '[0.0629, 0.2402, 0.1122, 0.0, 0.0899, 0.0, 0.0464, 0.0, 0.0127, 0.0, 0.0, 0.0319, 0.0, 0.0, 0.0, 0.0476]',\n", " 'redundancy': 0.3603,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0112,\n", " 'rank_mean': 3.039,\n", " 'killed': '',\n", " 'wall_s': 36.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m16_K4_free_s1234_52ba39',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6894659,\n", " 'acc': 0.3369,\n", " 'ce_bits': 2.8001,\n", " 'delivered_bits': 1.7952,\n", " 'cum_bits': 1.1018,\n", " 'marginal_bits': '[0.173, 0.0477, 0.0386, 0.3893, 0.0332, 0.0743, 0.1745, 0.019, 0.0, 0.0482, 0.0232, 0.0567, 0.0, 0.0057, 0.0183, 0.0]',\n", " 'redundancy': 0.1431,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0027,\n", " 'rank_mean': 2.951,\n", " 'killed': '',\n", " 'wall_s': 167.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6820547,\n", " 'acc': 0.1948,\n", " 'ce_bits': 2.8133,\n", " 'delivered_bits': 1.7675,\n", " 'cum_bits': 0.0789,\n", " 'marginal_bits': '[0.0789]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0068,\n", " 'rank_mean': 3.93,\n", " 'killed': '',\n", " 'wall_s': 111.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'sum_m16_K4_free_s5678_a631de',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6845523,\n", " 'acc': 0.325,\n", " 'ce_bits': 2.803,\n", " 'delivered_bits': 1.7864,\n", " 'cum_bits': 1.1082,\n", " 'marginal_bits': '[0.1179, 0.4659, 0.1057, 0.0702, 0.0594, 0.0243, 0.0, 0.1206, 0.0, 0.0087, 0.0766, 0.0376, 0.0, 0.0, 0.0214, 0.0]',\n", " 'redundancy': 0.3278,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0198,\n", " 'rank_mean': 3.064,\n", " 'killed': '',\n", " 'wall_s': 168.9,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'sum_m16_K4_free_s1234_85541a',\n", " 'op': 'sum',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6845523,\n", " 'acc': 0.3516,\n", " 'ce_bits': 2.8224,\n", " 'delivered_bits': 1.7882,\n", " 'cum_bits': 1.1666,\n", " 'marginal_bits': '[0.2555, 0.2261, 0.0993, 0.1091, 0.0491, 0.074, 0.0919, 0.0353, 0.0872, 0.0, 0.0028, 0.0282, 0.0326, 0.0258, 0.0261, 0.0235]',\n", " 'redundancy': 0.2295,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.001,\n", " 'rank_mean': 2.971,\n", " 'killed': '',\n", " 'wall_s': 168.9,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_free_s1234_9db65b',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6881347,\n", " 'acc': 0.333,\n", " 'ce_bits': 2.8342,\n", " 'delivered_bits': 1.7109,\n", " 'cum_bits': 1.0539,\n", " 'marginal_bits': '[0.1654, 0.1425, 0.0577, 0.5625, 0.0606, 0.0025, 0.0327, 0.0299]',\n", " 'redundancy': 0.1392,\n", " 'cancellation': 0.9915,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0169,\n", " 'rank_mean': 3.624,\n", " 'killed': '',\n", " 'wall_s': 137.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m16_K4_free_s5678_d700ae',\n", " 'op': 'res',\n", " 'm': 16,\n", " 'K': 4,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6894659,\n", " 'acc': 0.3611,\n", " 'ce_bits': 2.8267,\n", " 'delivered_bits': 1.7984,\n", " 'cum_bits': 1.2973,\n", " 'marginal_bits': '[0.4678, 0.2041, 0.0737, 0.3107, 0.0024, 0.0559, 0.0, 0.0757, 0.0, 0.0, 0.0638, 0.036, 0.0, 0.0017, 0.0, 0.0055]',\n", " 'redundancy': 0.195,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0124,\n", " 'rank_mean': 3.046,\n", " 'killed': '',\n", " 'wall_s': 167.4,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_free_s5678_0c1ab5',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6881347,\n", " 'acc': 0.3489,\n", " 'ce_bits': 2.8181,\n", " 'delivered_bits': 1.7739,\n", " 'cum_bits': 1.1865,\n", " 'marginal_bits': '[0.171, 0.3499, 0.1002, 0.0986, 0.0, 0.2152, 0.2516, 0.0]',\n", " 'redundancy': 0.0979,\n", " 'cancellation': 0.9848,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0025,\n", " 'rank_mean': 3.606,\n", " 'killed': '',\n", " 'wall_s': 137.4,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s1234_7106f9',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6832203,\n", " 'acc': 0.2959,\n", " 'ce_bits': 2.7934,\n", " 'delivered_bits': 1.8085,\n", " 'cum_bits': 0.8499,\n", " 'marginal_bits': '[0.2242, 0.1211, 0.0114, 0.3093, 0.0603, 0.0973, 0.0248, 0.0016]',\n", " 'redundancy': 0.1342,\n", " 'cancellation': 0.9955,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0167,\n", " 'rank_mean': 3.627,\n", " 'killed': '',\n", " 'wall_s': 138.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s5678_ccb02a',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6832203,\n", " 'acc': 0.2676,\n", " 'ce_bits': 2.8289,\n", " 'delivered_bits': 1.7783,\n", " 'cum_bits': 0.7585,\n", " 'marginal_bits': '[0.0698, 0.46, 0.0, 0.0248, 0.1357, 0.0, 0.0299, 0.0383]',\n", " 'redundancy': 0.1429,\n", " 'cancellation': 0.9986,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0019,\n", " 'rank_mean': 3.607,\n", " 'killed': '',\n", " 'wall_s': 138.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_free_s5678_800880',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6820547,\n", " 'acc': 0.2017,\n", " 'ce_bits': 2.7942,\n", " 'delivered_bits': 1.8023,\n", " 'cum_bits': 0.1033,\n", " 'marginal_bits': '[0.1033]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0062,\n", " 'rank_mean': 3.977,\n", " 'killed': '',\n", " 'wall_s': 111.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'}]" ] }, "metadata": {}, "execution_count": 11 } ] }, { "cell_type": "code", "source": [ "arms = generate_arms([\"prod\"], ms=[8], freezes=[\"free\"], seeds=[1234, 5678])\n", "fc = ForgeConfig(out_dir=\"exp011L\", push=True, rungs=((\"L\",1000,0.5),(\"L\",5000,1.0)),\n", " lm=tier_l_overrides())\n", "run_screen([a for a in arms if a.op==\"prod\"], fc, rungs=[1])" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6Lp0OGH832y3", "outputId": "c6ad7014-ee47-4b05-9353-45c7f27510b4" }, "execution_count": 12, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[TrigramStream] loading HF corpus wikitext-103-raw-v1 ...\n", "[TrigramStream] 100,000,000 bytes = 33,333,333 trigrams\n", "\n", "[rung 1] tier=L steps=5000 arms=2 (cached 0, fresh 2)\n", " [1/2] prod_m8_K8_free_s1234_e1d889 held 0.90 ce 2.814 dlv 1.84 [0.5557, 0.0032, 0.0977, 0.096, 0.1075, 0.0, 0.0, 0.037] acc 0.323 red 0.05\n", " [2/2] prod_m8_K8_free_s5678_51ec06 held 0.87 ce 2.818 dlv 1.75 [0.5958, 0.0, 0.0354, 0.0625, 0.0, 0.0367, 0.1196, 0.0237] acc 0.311 red 0.11\n", "[push] exp011L/ -> AbstractPhil/geolip-aleph-differentiation ✓\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "[{'arm_id': 'prod_m8_K8_free_s1234_e1d889',\n", " 'op': 'prod',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6968899,\n", " 'acc': 0.3225,\n", " 'ce_bits': 2.8139,\n", " 'delivered_bits': 1.8385,\n", " 'cum_bits': 0.897,\n", " 'marginal_bits': '[0.5557, 0.0032, 0.0977, 0.096, 0.1075, 0.0, 0.0, 0.037]',\n", " 'redundancy': 0.0547,\n", " 'cancellation': 0.9992,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0176,\n", " 'rank_mean': 3.627,\n", " 'killed': '',\n", " 'wall_s': 137.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 2/2 by cum_bits'},\n", " {'arm_id': 'prod_m8_K8_free_s5678_51ec06',\n", " 'op': 'prod',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6968899,\n", " 'acc': 0.311,\n", " 'ce_bits': 2.8184,\n", " 'delivered_bits': 1.7501,\n", " 'cum_bits': 0.8737,\n", " 'marginal_bits': '[0.5958, 0.0, 0.0354, 0.0625, 0.0, 0.0367, 0.1196, 0.0237]',\n", " 'redundancy': 0.1077,\n", " 'cancellation': 0.9957,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0009,\n", " 'rank_mean': 3.604,\n", " 'killed': '',\n", " 'wall_s': 135.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 2/2 by cum_bits'}]" ] }, "metadata": {}, "execution_count": 12 } ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_forge.py — exp_011 automation\n", "# Grammar -> generator (auto budget-twins + SUM controls) -> rung\n", "# scheduler (successive halving) -> kill rules -> ledger -> HF push.\n", "#\n", "# The Captain reviews VERDICTS, not arms: every promote/park/kill is\n", "# logged with the gauge values that caused it.\n", "#\n", "# Rungs v1: P-200 -> P-1000 (Tier-P implemented). Tier-L rungs raise\n", "# NotImplementedError at a clean seam until acd_lm_adapter.py lands.\n", "# Lane parallelism (vmap seed-groups): DEFERRED v1.1 — sequential is\n", "# correct and rung0 runs single-seed; the bit-equivalence gate applies\n", "# when lanes ship, not before.\n", "#\n", "# Repo: AbstractPhil/geolip-aleph-differentiation (exp011/ prefix)\n", "# Paste order: acd_structures.py -> acd_probe.py -> acd_forge.py\n", "# ============================================================\n", "from __future__ import annotations\n", "import csv, hashlib, json, math, os, time\n", "from dataclasses import asdict, dataclass, field\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure, match_budget, OPS\n", " from acd_probe import (BubbleConfig, NestedBubbles, marginal_bits,\n", " composition_gauges)\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver. Pasted Colab cells share ONE namespace and are\n", " not importable modules — so resolve names from globals() first (paste\n", " mode), then fall back to a real import (script/module mode).\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "REPO_ID = \"AbstractPhil/geolip-aleph-differentiation\"\n", "EXP_PREFIX = \"exp011\"\n", "BUDGET_KD = 64 * 4 # single-aleph reference: K=64, D=4 codebook floats\n", "\n", "@dataclass\n", "class ForgeConfig:\n", " out_dir: str = \"exp011\"\n", " rungs: Tuple[Tuple[str, int, float], ...] = (\n", " (\"P\", 200, 1 / 3), (\"P\", 1000, 1 / 3),\n", " (\"L\", 1000, 1 / 3), (\"L\", 5000, 1.0))\n", " lr: float = 3e-3\n", " batch: int = 512\n", " eval_every: int = 50\n", " probe_steps: int = 250 # marginal-bits probe budget per prefix\n", " n_train: int = 8192 # fixed shared dataset per screen\n", " n_eval: int = 4096\n", " bubble: \"BubbleConfig\" = None # set in __post_init__ (shared task!)\n", " push: bool = True # False = dry run (no network)\n", " lm: Optional[Dict] = None # Tier-L AlephLM overrides (None -> tier_l recipe)\n", " lm_corpus: str = \"wikitext-103-raw-v1\" # 'synthetic' -> Markov smoke stream\n", " device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " seed: int = 1234\n", "\n", " exp_prefix: str = \"\" # HF push prefix; defaults to out_dir\n", "\n", " def __post_init__(self):\n", " if not self.exp_prefix:\n", " self.exp_prefix = self.out_dir\n", " if self.bubble is None:\n", " self.bubble = BubbleConfig(d_data=32, branching=(4, 4, 4),\n", " sep0=6.0, sep_decay=0.45,\n", " noise=0.35, seed=97)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Grammar — an arm is a JSON spec; its hash is its identity\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class ArmSpec:\n", " op: str\n", " m: int\n", " d_addr: int = 4\n", " freeze: str = \"free\"\n", " seed: int = 1234\n", " tree_hard: bool = False\n", " budget_kd: int = BUDGET_KD\n", " K: int = 0 # 0 -> solved by match_budget\n", " coupling: str = \"whisper\" # Tier-L head coupling (3b ladder)\n", "\n", " def resolve(self) -> \"ArmSpec\":\n", " if self.K == 0:\n", " self.K = match_budget(self.op, self.m, self.d_addr,\n", " self.budget_kd)\n", " if self.op == \"single\":\n", " self.m = 1\n", " return self\n", "\n", " def arm_id(self) -> str:\n", " d = asdict(self)\n", " if d.get(\"coupling\", \"whisper\") == \"whisper\":\n", " d.pop(\"coupling\", None) # LEGACY-STABLE: old ids reproduce\n", " s = json.dumps(d, sort_keys=True)\n", " cpl = \"\" if self.coupling == \"whisper\" else f\"_{self.coupling}\"\n", " return f\"{self.op}_m{self.m}_K{self.K}\" \\\n", " f\"{'_hard' if self.tree_hard else ''}{cpl}\" \\\n", " f\"_{self.freeze}_s{self.seed}_{hashlib.sha1(s.encode()).hexdigest()[:6]}\"\n", "\n", " def to_acd(self, d_in: int, feature_dim: int = 64) -> \"ACDConfig\":\n", " return ACDConfig(op=self.op, d_in=d_in, m=self.m, K=self.K,\n", " d_addr=self.d_addr, freeze=self.freeze,\n", " tree_hard=self.tree_hard, seed=self.seed,\n", " feature_dim=feature_dim)\n", "\n", "\n", "def generate_arms(ops: List[str], ms: List[int], freezes: List[str],\n", " seeds: List[int], existing_ids: Optional[set] = None,\n", " tree_both_modes: bool = True) -> List[ArmSpec]:\n", " \"\"\"Grid expansion + the two mandatory scientific controls:\n", " (1) budget-matched SINGLE twin per (freeze, seed),\n", " (2) SUM control at every m present (the divergence reference).\n", " Dedup against existing ledger ids.\"\"\"\n", " arms: List[ArmSpec] = []\n", " for op in ops:\n", " for m in ms:\n", " if op == \"single\":\n", " continue\n", " for fz in freezes:\n", " for sd in seeds:\n", " if op == \"tree\" and tree_both_modes:\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd,\n", " tree_hard=False))\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd,\n", " tree_hard=True))\n", " else:\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd))\n", " for fz in freezes: # control (1)\n", " for sd in seeds:\n", " arms.append(ArmSpec(\"single\", 1, freeze=fz, seed=sd))\n", " if \"sum\" not in ops: # control (2)\n", " for m in ms:\n", " for fz in freezes:\n", " for sd in seeds:\n", " arms.append(ArmSpec(\"sum\", m, freeze=fz, seed=sd))\n", " out, seen = [], set(existing_ids or ())\n", " for a in arms:\n", " a.resolve()\n", " if a.arm_id() not in seen:\n", " seen.add(a.arm_id())\n", " out.append(a)\n", " return out\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Kill rules — fire inside a rung; every kill carries its reason\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def check_kill(loss_val: float, grad_norm: float,\n", " stats: List[Dict]) -> Optional[str]:\n", " if not math.isfinite(loss_val):\n", " return \"NaN/inf loss\"\n", " if grad_norm > 1e3:\n", " return f\"grad blowup |g|={grad_norm:.1f}\"\n", " for i, s in enumerate(stats):\n", " ceiling = min(4.0, s.get(\"eff_rank_ceiling\", 4.0))\n", " if s[\"eff_rank\"] < 0.5 * ceiling:\n", " return f\"rank collapse stage{i} rank={s['eff_rank']:.2f}\"\n", " return None\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier-P trainer — one arm, one rung\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_arm_P(spec: ArmSpec, steps: int, fc: ForgeConfig,\n", " data: Tuple) -> Dict[str, object]:\n", " xtr, ytr, ltr, xev, yev, lev = data\n", " dev = fc.device\n", " torch.manual_seed(spec.seed)\n", " net = ACDStructure(spec.to_acd(d_in=fc.bubble.d_data)).to(dev)\n", " head = nn.Linear(net.cfg.feature_dim, fc.bubble.n_leaves).to(dev)\n", " opt = torch.optim.Adam(list(net.parameters()) + list(head.parameters()),\n", " lr=fc.lr) # pure Adam, statute\n", " t0, killed = time.time(), None\n", " n = xtr.shape[0]\n", " for step in range(1, steps + 1):\n", " idx = torch.randint(0, n, (fc.batch,), device=dev)\n", " feats, _ = net(xtr[idx])\n", " loss = F.cross_entropy(head(feats), ytr[idx])\n", " opt.zero_grad()\n", " loss.backward()\n", " gn = torch.nn.utils.clip_grad_norm_(\n", " list(net.parameters()) + list(head.parameters()),\n", " max(loss.item(), 1.0)) # standing clip rule\n", " opt.step()\n", " if step % fc.eval_every == 0 or step == steps:\n", " reason = check_kill(loss.item(), gn.item(), net.codebook_stats())\n", " if reason:\n", " killed = reason\n", " break\n", " # rung-end gauges (also computed for killed arms — the corpse is data)\n", " net.eval()\n", " with torch.no_grad():\n", " fe, _ = net(xev)\n", " ce_bits = F.cross_entropy(head(fe), yev).item() / math.log(2)\n", " acc = (head(fe).argmax(-1) == yev).float().mean().item()\n", " # probe budget scales with class count: 250 steps underfits a\n", " # 256-way linear readout (measured: single held 3.11 vs delivered\n", " # 5.34 at 2b). Within-op comparisons survive the bias; the\n", " # held-vs-delivered gap does not, so we feed the probe properly.\n", " psteps = max(fc.probe_steps, int(2.5 * fc.bubble.n_leaves))\n", " mb = marginal_bits(net, xev, yev, fc.bubble.n_leaves,\n", " probe_steps=psteps, seed=spec.seed)\n", " cg = composition_gauges(net, xev[:1024], cluster=lev[:1024, 0])\n", " st = net.codebook_stats()\n", " row = dict(arm_id=spec.arm_id(), op=spec.op, m=spec.m, K=spec.K,\n", " d_addr=spec.d_addr, freeze=spec.freeze, seed=spec.seed,\n", " tree_hard=spec.tree_hard, steps=steps,\n", " params=net.param_count(),\n", " acc=round(acc, 4), ce_bits=round(ce_bits, 4),\n", " cum_bits=round(mb[\"cumulative_bits\"][-1], 4),\n", " delivered_bits=round(\n", " math.log2(fc.bubble.n_leaves) - ce_bits, 4),\n", " marginal_bits=json.dumps(\n", " [round(v, 4) for v in mb[\"marginal_bits\"]]),\n", " redundancy=round(cg[\"redundancy\"], 4),\n", " cancellation=round(cg[\"cancellation\"], 4),\n", " stage_snr=round(cg.get(\"stage_snr\", 0.0), 4),\n", " dev_mean=round(sum(s[\"deviation\"] for s in st) / len(st), 4),\n", " rank_mean=round(sum(s[\"eff_rank\"] for s in st) / len(st), 3),\n", " killed=killed or \"\", wall_s=round(time.time() - t0, 1))\n", " return row\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Ledger + HF push\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "_CSV_COLS = [\"rung\", \"arm_id\", \"op\", \"m\", \"K\", \"d_addr\", \"freeze\", \"seed\",\n", " \"tree_hard\", \"steps\", \"params\", \"acc\", \"ce_bits\", \"cum_bits\",\n", " \"delivered_bits\",\n", " \"marginal_bits\", \"redundancy\", \"cancellation\", \"stage_snr\",\n", " \"dev_mean\", \"rank_mean\", \"alpha\",\n", " \"verdict\", \"reason\", \"killed\", \"wall_s\"]\n", "\n", "def ledger_append(fc: ForgeConfig, rows: List[Dict]):\n", " os.makedirs(fc.out_dir, exist_ok=True)\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " new = not os.path.exists(path)\n", " if new:\n", " cols = _CSV_COLS\n", " else: # header-aware: never shift columns of a pre-existing ledger\n", " with open(path) as f:\n", " cols = next(csv.reader(f))\n", " with open(path, \"a\", newline=\"\") as f:\n", " w = csv.DictWriter(f, fieldnames=cols, extrasaction=\"ignore\",\n", " restval=\"\")\n", " if new:\n", " w.writeheader()\n", " w.writerows(rows)\n", "\n", "\n", "def ledger_rows(fc: ForgeConfig) -> List[Dict]:\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " if not os.path.exists(path):\n", " return []\n", " with open(path) as f:\n", " return list(csv.DictReader(f))\n", "\n", "\n", "def ledger_ids(fc: ForgeConfig) -> set:\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " if not os.path.exists(path):\n", " return set()\n", " with open(path) as f:\n", " return {r[\"arm_id\"] for r in csv.DictReader(f)}\n", "\n", "\n", "def write_sweep_md(fc: ForgeConfig, all_rows: List[Dict]):\n", " rows = sorted(all_rows, key=lambda r: -float(r[\"cum_bits\"]))\n", " lines = [\"# exp_011 ACD — sweep leaderboard\", \"\",\n", " f\"Task: nested bubbles {fc.bubble.branching} \"\n", " f\"({fc.bubble.n_leaves} leaves, \"\n", " f\"{math.log2(fc.bubble.n_leaves):.1f} bits available)\", \"\",\n", " \"| arm | rung | cum bits | marginal | acc | red | canc | dev | rank | verdict |\",\n", " \"|---|---|---|---|---|---|---|---|---|---|\"]\n", " for r in rows:\n", " lines.append(\n", " f\"| `{r['arm_id']}` | {r['rung']} | **{r['cum_bits']}** \"\n", " f\"| {r['marginal_bits']} | {r['acc']} | {r['redundancy']} \"\n", " f\"| {r['cancellation']} | {r['dev_mean']} | {r['rank_mean']} \"\n", " f\"| {r['verdict']}{(' — ' + r['reason']) if r['reason'] else ''} |\")\n", " with open(os.path.join(fc.out_dir, \"SWEEP.md\"), \"w\") as f:\n", " f.write(\"\\n\".join(lines) + \"\\n\")\n", "\n", "\n", "def hf_push(fc: ForgeConfig):\n", " if not fc.push:\n", " print(\"[push] dry run — skipped\")\n", " return\n", " from huggingface_hub import HfApi, create_repo\n", " create_repo(REPO_ID, repo_type=\"model\", exist_ok=True)\n", " api = HfApi()\n", " for name in (\"results.csv\", \"SWEEP.md\", \"verdicts.jsonl\"):\n", " p = os.path.join(fc.out_dir, name)\n", " if os.path.exists(p):\n", " api.upload_file(path_or_fileobj=p,\n", " path_in_repo=f\"{fc.exp_prefix}/{name}\",\n", " repo_id=REPO_ID, repo_type=\"model\",\n", " commit_message=f\"{fc.exp_prefix}: {name}\")\n", " print(f\"[push] {fc.exp_prefix}/ -> {REPO_ID} ✓\")\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Scheduler — successive halving with logged verdicts\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_screen(arms: List[ArmSpec], fc: ForgeConfig,\n", " rungs: Optional[List[int]] = None,\n", " protect_ops: Tuple[str, ...] = ()) -> List[Dict]:\n", " \"\"\"Run arms through the configured rungs; keep top keep_frac by\n", " cum_bits per rung (kills never advance). Returns all ledger rows.\"\"\"\n", " g = torch.Generator().manual_seed(fc.seed)\n", " bub = NestedBubbles(fc.bubble)\n", " xtr, ytr, ltr = bub.sample(fc.n_train, device=fc.device)\n", " xev, yev, lev = bub.sample(fc.n_eval, device=fc.device)\n", " data = (xtr, ytr, ltr, xev, yev, lev)\n", "\n", " alive = list(arms)\n", " if not alive:\n", " prior = ledger_rows(fc)\n", " print(f\"[screen] nothing queued — ledger already holds \"\n", " f\"{len(prior)} rows in {fc.out_dir}/results.csv; \"\n", " f\"call report(fc) to view standings.\")\n", " return prior\n", " all_rows: List[Dict] = []\n", " vpath = os.path.join(fc.out_dir, \"verdicts.jsonl\")\n", " os.makedirs(fc.out_dir, exist_ok=True)\n", " for ri, (tier, steps, keep) in enumerate(fc.rungs):\n", " if rungs is not None and ri not in rungs:\n", " continue\n", " if tier == \"L\":\n", " try:\n", " run_arm_L = _ns(\"run_arm_L\", \"acd_lm_adapter\")\n", " tier_l_overrides = _ns(\"tier_l_overrides\",\n", " \"acd_lm_adapter\")\n", " MarkovByteStream = _ns(\"MarkovByteStream\",\n", " \"acd_lm_adapter\")\n", " except Exception as e:\n", " raise RuntimeError(\n", " \"Tier-L needs acd_lm_adapter in the namespace — \"\n", " \"paste it (after the aleph-lm cells) first.\") from e\n", " if not hasattr(fc, \"_lm_stream\"):\n", " if fc.lm_corpus == \"synthetic\":\n", " fc._lm_stream = MarkovByteStream(seed=fc.seed)\n", " else:\n", " TrigramStream = _ns(\"TrigramStream\", \"aleph_lm\")\n", " fc._lm_stream = TrigramStream(\n", " fc.lm_corpus, max_corpus_bytes=100_000_000,\n", " seed=fc.seed)\n", " lm_over = fc.lm or tier_l_overrides()\n", " run_L = lambda sp, st: run_arm_L(\n", " sp, st, fc._lm_stream, lm_over, fc.device,\n", " probe_steps=max(fc.probe_steps, 640),\n", " eval_every=fc.eval_every)\n", " cached = {r[\"arm_id\"]: r for r in ledger_rows(fc)\n", " if r.get(\"rung\") == str(ri)}\n", " n_hit = sum(1 for s in alive if s.arm_id() in cached)\n", " print(f\"\\n[rung {ri}] tier={tier} steps={steps} \"\n", " f\"arms={len(alive)} (cached {n_hit}, \"\n", " f\"fresh {len(alive) - n_hit})\")\n", " rows, fresh = [], []\n", " for k, spec in enumerate(alive):\n", " aid = spec.arm_id()\n", " if aid in cached: # RESUME: reuse the ledger row\n", " rows.append(cached[aid])\n", " print(f\" [{k+1}/{len(alive)}] {aid:34s} cached \"\n", " f\"(bits {cached[aid]['cum_bits']})\")\n", " continue\n", " row = (run_arm_P(spec, steps, fc, data) if tier == \"P\"\n", " else run_L(spec, steps))\n", " row[\"rung\"] = ri\n", " rows.append(row)\n", " fresh.append(row)\n", " print(f\" [{k+1}/{len(alive)}] {row['arm_id']:34s} \"\n", " f\"held {row['cum_bits']:.2f} ce {row['ce_bits']:.3f} \"\n", " f\"dlv {row['delivered_bits']:.2f} \"\n", " f\"{row['marginal_bits']} \"\n", " f\"acc {row['acc']:.3f} red {row['redundancy']:.2f}\"\n", " f\"{' a ' + format(row['alpha'], '.3f') if 'alpha' in row else ''}\"\n", " f\"{' KILLED: ' + row['killed'] if row['killed'] else ''}\")\n", " # verdicts\n", " survivors = [r for r in rows if not r[\"killed\"]]\n", " survivors.sort(key=lambda r: -float(r[\"cum_bits\"]))\n", " n_keep = max(1, int(len(survivors) * keep))\n", " promoted = {r[\"arm_id\"] for r in survivors[:n_keep]}\n", " # science over leaderboard: controls ride every rung so the\n", " # divergence gate is actually tested at full training depth\n", " shielded = {r[\"arm_id\"] for r in survivors\n", " if r[\"op\"] in protect_ops}\n", " promoted |= shielded\n", " with open(vpath, \"a\") as vf:\n", " for r in rows:\n", " if r[\"killed\"]:\n", " r[\"verdict\"], r[\"reason\"] = \"KILL\", r[\"killed\"]\n", " elif r[\"arm_id\"] in promoted:\n", " why = (\"protected control\"\n", " if r[\"op\"] in protect_ops\n", " and r[\"arm_id\"] not in\n", " {s[\"arm_id\"] for s in survivors[:n_keep]}\n", " else f\"top {n_keep}/{len(survivors)} by cum_bits\")\n", " r[\"verdict\"], r[\"reason\"] = \"PROMOTE\", why\n", " else:\n", " r[\"verdict\"], r[\"reason\"] = \"PARK\", \"below keep line\"\n", " if any(r is fr for fr in fresh):\n", " vf.write(json.dumps(r) + \"\\n\")\n", " ledger_append(fc, fresh)\n", " all_rows += rows\n", " write_sweep_md(fc, all_rows)\n", " hf_push(fc)\n", " by_id = {a.arm_id(): a for a in alive}\n", " alive = [by_id[i] for i in promoted if i in by_id]\n", " if not alive:\n", " print(\"[screen] no survivors — stopping\")\n", " break\n", " return all_rows\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Activation — dual-mode (script main / notebook cell)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def phase2_screen(push: bool = True):\n", " \"\"\"The Phase-2 mass screen: full operator grid, rung0+rung1.\"\"\"\n", " fc = ForgeConfig(push=push)\n", " ops = [\"sum\", \"gate\", \"res\", \"prod\", \"tree\", \"cross\", \"anneal\"]\n", " arms = generate_arms(ops, ms=[2, 3, 4], freezes=[\"free\", \"spread\"],\n", " seeds=[1234]) # resume-safe: cache skips reruns\n", " print(f\"[forge] {len(arms)} arms queued (incl. controls)\")\n", " return run_screen(arms, fc, rungs=[0, 1])\n", "\n", "\n", "def phase2b_screen(push: bool = True):\n", " \"\"\"Phase 2b: divergence hunt with headroom. 256-leaf task (8 bits),\n", " m up to 8, two seeds, SUM + SINGLE protected to full training depth.\n", " Own ledger (exp011b/) — the task changed, so rows must not mix.\"\"\"\n", " fc = ForgeConfig(out_dir=\"exp011b\", push=push,\n", " n_train=16384, n_eval=8192, batch=1024,\n", " bubble=BubbleConfig(d_data=32, branching=(4, 4, 4, 4),\n", " sep0=6.0, sep_decay=0.5,\n", " noise=0.3, seed=97))\n", " ops = [\"sum\", \"gate\", \"res\", \"prod\", \"cross\"] # anneal parked (rung-0\n", " # verdict: clone stages); tree awaits the budget-accounting decision\n", " arms = generate_arms(ops, ms=[2, 4, 6, 8], freezes=[\"free\", \"spread\"],\n", " seeds=[1234, 5678])\n", " print(f\"[forge] {len(arms)} arms queued (sum+single protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", "\n", "\n", "def tree_dual_arms(ms, freezes, seeds, budget_kd: int = BUDGET_KD,\n", " K0: int = 8, d_addr: int = 4):\n", " \"\"\"The 'both' ruling: every tree config in BOTH accountings.\n", " total — storage-matched: K = match_budget('tree', ...) (soft+hard)\n", " active — MoE convention (HARD only; soft touches all branches):\n", " per-input params = router K0*D + ONE branch K*D\n", " => K_active = (budget - K0*D) // d_addr.\"\"\"\n", " arms = []\n", " K_act = max(2, (budget_kd - K0 * d_addr) // d_addr)\n", " for m in ms:\n", " for fz in freezes:\n", " for sd in seeds:\n", " for hard in (False, True): # total accounting\n", " arms.append(ArmSpec(\"tree\", m, d_addr=d_addr, freeze=fz,\n", " seed=sd, tree_hard=hard).resolve())\n", " arms.append(ArmSpec(\"tree\", m, d_addr=d_addr, freeze=fz,\n", " seed=sd, tree_hard=True,\n", " K=K_act)) # active accounting (hard)\n", " return arms\n", "\n", "\n", "def phase3b_screen(push: bool = True):\n", " \"\"\"Phase 3b — head-through-structure: the coupling ladder on the\n", " language champion (res_m8), sum control + singles protected.\n", " apmix_src arms run the hybrid recipe (corpus bank); the rest byte-head.\n", " Own ledger exp011H.\"\"\"\n", " tier_l_overrides = _ns(\"tier_l_overrides\", \"acd_lm_adapter\")\n", " hybrid = _ns(\"tier_l_hybrid_overrides\", \"acd_lm_adapter\")\n", " fc = ForgeConfig(out_dir=\"exp011H\", push=push,\n", " rungs=((\"L\", 1000, 0.6), (\"L\", 5000, 1.0)),\n", " lm=tier_l_overrides())\n", " fc_h = ForgeConfig(out_dir=\"exp011H\", push=push,\n", " rungs=fc.rungs, lm=hybrid())\n", " arms, arms_h = [], []\n", " for sd in (1234, 5678):\n", " for cpl in (\"whisper\", \"concat\", \"gate\"):\n", " arms.append(ArmSpec(\"res\", 8, freeze=\"free\", seed=sd,\n", " coupling=cpl).resolve())\n", " arms.append(ArmSpec(\"sum\", 8, freeze=\"free\", seed=sd).resolve())\n", " arms.append(ArmSpec(\"single\", 1, freeze=\"free\", seed=sd).resolve())\n", " arms_h.append(ArmSpec(\"res\", 8, freeze=\"free\", seed=sd,\n", " coupling=\"apmix_src\").resolve())\n", " print(f\"[forge] 3b: {len(arms)} byte-head + {len(arms_h)} hybrid arms\")\n", " rows = run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", " rows += run_screen(arms_h, fc_h, rungs=[0, 1])\n", " return rows\n", "\n", "\n", "def phase3_screen(push: bool = True):\n", " \"\"\"Phase 3: Tier-P finalists onto the 6.75M LM (WikiText-103).\n", " prod/res at their knees + sum control + singles, byte head first\n", " (bank-free); hybrid/apmix arms follow once the byte read is banked.\n", " ce_bits column = bits/byte; delivered_bits = 8 - bpb.\"\"\"\n", " tier_l_overrides = _ns(\"tier_l_overrides\", \"acd_lm_adapter\")\n", " fc = ForgeConfig(out_dir=\"exp011L\", push=push,\n", " rungs=((\"L\", 1000, 0.5), (\"L\", 5000, 1.0)),\n", " lm=tier_l_overrides())\n", " arms = generate_arms([\"prod\", \"res\", \"sum\"], ms=[8],\n", " freezes=[\"free\"], seeds=[1234, 5678])\n", " arms += generate_arms([\"res\"], ms=[16], freezes=[\"free\"],\n", " seeds=[1234, 5678],\n", " existing_ids={a.arm_id() for a in arms})\n", " print(f\"[forge] {len(arms)} Tier-L arms queued (sum+single protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", "\n", "\n", "def report(fc: Optional[ForgeConfig] = None, top: int = 15):\n", " \"\"\"Print standings from the ledger without running anything.\"\"\"\n", " fc = fc or ForgeConfig(push=False)\n", " rows = ledger_rows(fc)\n", " if not rows:\n", " print(f\"[report] no ledger at {fc.out_dir}/results.csv\")\n", " return rows\n", " rows.sort(key=lambda r: (-int(r[\"rung\"]), -float(r[\"cum_bits\"])))\n", " print(f\"[report] {len(rows)} rows — top {top}:\")\n", " for r in rows[:top]:\n", " k = \" KILLED:\" + r[\"killed\"] if r[\"killed\"] else \"\"\n", " print(f\" r{r['rung']} {r['arm_id']:36s} bits {r['cum_bits']:>7s} \"\n", " f\"acc {r['acc']:>6s} red {r['redundancy']:>6s} \"\n", " f\"{r['verdict']}{k}\")\n", " return rows\n", "\n", "\n", "def _smoke():\n", " # Self-cleaning: Colab working dirs PERSIST across sessions, and a\n", " # stale smoke ledger turns fresh-arm tests into cache hits. The\n", " # smoke always starts from a swept room; resume behavior is tested\n", " # WITHIN the smoke (first pass trains, second pass must cache).\n", " import shutil\n", " for _d in (\"exp011_smoke\", \"exp011_smoke_prot\"):\n", " shutil.rmtree(_d, ignore_errors=True)\n", " fc = ForgeConfig(push=False, n_train=1536, n_eval=768, batch=256,\n", " probe_steps=120, eval_every=25,\n", " rungs=((\"P\", 60, 0.5), (\"P\", 120, 1.0)),\n", " out_dir=\"exp011_smoke\")\n", " arms = generate_arms([\"sum\", \"res\", \"prod\"], ms=[3], freezes=[\"free\"],\n", " seeds=[1234])\n", " ids = [a.arm_id() for a in arms]\n", " assert len(ids) == len(set(ids)), \"id collision\"\n", " assert any(a.op == \"single\" for a in arms), \"budget twin missing\"\n", " print(f\" ✓ generator: {len(arms)} arms \"\n", " f\"({[a.op for a in arms]})\")\n", " # kill-rule unit checks\n", " assert check_kill(float(\"nan\"), 1.0, []) == \"NaN/inf loss\"\n", " assert check_kill(1.0, 5e3, []).startswith(\"grad blowup\")\n", " assert check_kill(1.0, 1.0, [{\"eff_rank\": 1.2}]).startswith(\"rank collapse\")\n", " assert check_kill(1.0, 1.0, [{\"eff_rank\": 3.8}]) is None\n", " print(\" ✓ kill rules fire correctly\")\n", " rows = run_screen(arms, fc, rungs=[0, 1])\n", " assert os.path.exists(os.path.join(fc.out_dir, \"results.csv\"))\n", " assert os.path.exists(os.path.join(fc.out_dir, \"SWEEP.md\"))\n", " assert os.path.exists(os.path.join(fc.out_dir, \"verdicts.jsonl\"))\n", " assert all(r[\"verdict\"] in (\"PROMOTE\", \"PARK\", \"KILL\") for r in rows)\n", " # RESUME test: re-run same arms -> all cached, nothing trains;\n", " # add one new op -> only it trains, union re-ranked\n", " n_ledger_before = len(ledger_rows(fc))\n", " t0 = time.time()\n", " rows2 = run_screen(arms, fc, rungs=[0, 1])\n", " assert time.time() - t0 < 20, \"resume re-trained instead of caching\"\n", " assert len(ledger_rows(fc)) == n_ledger_before, \"cache appended rows\"\n", " r0 = [r for r in rows2 if str(r[\"rung\"]) == \"0\"]\n", " assert r0 and all(isinstance(r[\"cum_bits\"], str) for r in r0), \\\n", " \"rung0 rows were re-trained (fresh rows are floats)\"\n", " arms3 = generate_arms([\"sum\", \"res\", \"prod\", \"gate\"], ms=[3],\n", " freezes=[\"free\"], seeds=[1234])\n", " rows3 = run_screen(arms3, fc, rungs=[0])\n", " fresh_gate = [r for r in rows3 if r[\"op\"] == \"gate\"]\n", " assert len(fresh_gate) == 1 and isinstance(fresh_gate[0][\"cum_bits\"], float)\n", " assert isinstance(fresh_gate[0].get(\"delivered_bits\"), float), \\\n", " \"delivered_bits missing from fresh rows\"\n", " print(\" ✓ resume: full-cache fast path + mixed cache/fresh both work\")\n", " # protection: sum rides to rung1 despite bottom rank\n", " fcp = ForgeConfig(push=False, n_train=1024, n_eval=512, batch=256,\n", " probe_steps=80, eval_every=25,\n", " rungs=((\"P\", 40, 0.34), (\"P\", 60, 1.0)),\n", " out_dir=\"exp011_smoke_prot\")\n", " armsp = generate_arms([\"sum\", \"res\", \"prod\"], ms=[3], freezes=[\"free\"],\n", " seeds=[1234])\n", " rowsp = run_screen(armsp, fcp, rungs=[0, 1], protect_ops=(\"sum\",))\n", " r1_ops = {r[\"op\"] for r in rowsp if str(r[\"rung\"]) == \"1\"}\n", " assert \"sum\" in r1_ops, f\"protected sum culled: rung1 ops {r1_ops}\"\n", " prot = [r for r in rowsp if r[\"op\"] == \"sum\" and str(r[\"rung\"]) == \"0\"]\n", " assert any(\"protected\" in r[\"reason\"] or \"top\" in r[\"reason\"]\n", " for r in prot)\n", " print(\" ✓ protected controls ride the keep-line\")\n", " # Tier-L path (synthetic stream; skipped if adapter not pasted yet)\n", " try:\n", " smoke_overrides = _ns(\"smoke_overrides\", \"acd_lm_adapter\")\n", " import shutil as _sh\n", " _sh.rmtree(\"exp011_smoke_L\", ignore_errors=True)\n", " fcl = ForgeConfig(out_dir=\"exp011_smoke_L\", push=False,\n", " rungs=((\"L\", 25, 1.0),), probe_steps=150,\n", " eval_every=20, lm=smoke_overrides(),\n", " lm_corpus=\"synthetic\")\n", " armsl = generate_arms([\"prod\"], ms=[2], freezes=[\"free\"],\n", " seeds=[1234])\n", " rowsl = run_screen(armsl, fcl, rungs=[0])\n", " fr = [r for r in rowsl if isinstance(r[\"ce_bits\"], float)]\n", " assert fr and all(r[\"ce_bits\"] < 8.0 for r in fr)\n", " assert all(isinstance(r[\"delivered_bits\"], float) for r in fr)\n", " print(\" ✓ Tier-L rung: bpb finite, delivered logged, \"\n", " \"ledger schema intact\")\n", " except Exception:\n", " print(\" - Tier-L smoke skipped (acd_lm_adapter not in namespace)\")\n", " # empty-queue guard returns ledger instead of []\n", " empty = run_screen([], fc, rungs=[0])\n", " assert len(empty) > 0, \"empty guard returned nothing\"\n", " print(\" ✓ empty-queue guard returns ledger standings\")\n", " print(\"acd_forge smoke: ALL GREEN — next: phase2_screen()\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " # Notebook cells execute as __main__, so the smoke fires on paste too —\n", " # deliberate: pasting a cell IS the verification step in the Colab flow\n", " # (shared namespace, paste order structures -> probe -> forge).\n", " # Heavy entry points (phase2_screen) are never wired here; call them.\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "o0vFrBNpwxlW", "outputId": "0dcf5714-5444-406c-8770-947ce3729bdb" }, "execution_count": 8, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ generator: 4 arms (['sum', 'res', 'prod', 'single'])\n", " ✓ kill rules fire correctly\n", "\n", "[rung 0] tier=P steps=60 arms=4 (cached 0, fresh 4)\n", " [1/4] sum_m3_K21_free_s1234_7faffe held 3.68 ce 4.499 dlv 1.50 [2.9094, 0.2359, 0.5361] acc 0.111 red 0.23\n", " [2/4] res_m3_K21_free_s1234_1925f3 held 4.15 ce 3.174 dlv 2.83 [3.3556, 0.216, 0.5828] acc 0.310 red 0.08\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 held 4.13 ce 3.063 dlv 2.94 [3.0398, 0.5619, 0.5312] acc 0.253 red 0.05\n", " [4/4] single_m1_K64_free_s1234_194318 held 2.95 ce 4.330 dlv 1.67 [2.9487] acc 0.185 red 0.00\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=120 arms=2 (cached 0, fresh 2)\n", " [1/2] prod_m3_K21_free_s1234_c5d5b7 held 4.36 ce 1.970 dlv 4.03 [3.2752, 0.5116, 0.5715] acc 0.453 red 0.05\n", " [2/2] res_m3_K21_free_s1234_1925f3 held 4.41 ce 1.938 dlv 4.06 [3.4941, 0.2551, 0.6582] acc 0.466 red 0.06\n", "[push] dry run — skipped\n", "\n", "[rung 0] tier=P steps=60 arms=4 (cached 4, fresh 0)\n", " [1/4] sum_m3_K21_free_s1234_7faffe cached (bits 3.6814)\n", " [2/4] res_m3_K21_free_s1234_1925f3 cached (bits 4.1544)\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.1329)\n", " [4/4] single_m1_K64_free_s1234_194318 cached (bits 2.9487)\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=120 arms=2 (cached 2, fresh 0)\n", " [1/2] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.3582)\n", " [2/2] res_m3_K21_free_s1234_1925f3 cached (bits 4.4074)\n", "[push] dry run — skipped\n", "\n", "[rung 0] tier=P steps=60 arms=5 (cached 4, fresh 1)\n", " [1/5] sum_m3_K21_free_s1234_7faffe cached (bits 3.6814)\n", " [2/5] res_m3_K21_free_s1234_1925f3 cached (bits 4.1544)\n", " [3/5] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.1329)\n", " [4/5] gate_m3_K21_free_s1234_0d3f58 held 3.54 ce 4.379 dlv 1.62 [2.7871, 0.3954, 0.3608] acc 0.134 red 0.27\n", " [5/5] single_m1_K64_free_s1234_194318 cached (bits 2.9487)\n", "[push] dry run — skipped\n", " ✓ resume: full-cache fast path + mixed cache/fresh both work\n", "\n", "[rung 0] tier=P steps=40 arms=4 (cached 0, fresh 4)\n", " [1/4] sum_m3_K21_free_s1234_7faffe held 3.46 ce 5.407 dlv 0.59 [2.4185, 0.8562, 0.1877] acc 0.123 red 0.18\n", " [2/4] res_m3_K21_free_s1234_1925f3 held 3.72 ce 4.209 dlv 1.79 [2.8238, 0.6729, 0.2209] acc 0.203 red 0.11\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 held 3.67 ce 4.186 dlv 1.81 [2.6164, 0.6287, 0.4231] acc 0.221 red 0.07\n", " [4/4] single_m1_K64_free_s1234_194318 held 2.47 ce 5.279 dlv 0.72 [2.4703] acc 0.166 red 0.00\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=60 arms=2 (cached 0, fresh 2)\n", " [1/2] sum_m3_K21_free_s1234_7faffe held 3.38 ce 4.597 dlv 1.40 [2.5272, 0.7302, 0.1235] acc 0.135 red 0.22\n", " [2/2] res_m3_K21_free_s1234_1925f3 held 3.80 ce 3.152 dlv 2.85 [2.9599, 0.6163, 0.2213] acc 0.268 red 0.09\n", "[push] dry run — skipped\n", " ✓ protected controls ride the keep-line\n", " - Tier-L smoke skipped (acd_lm_adapter not in namespace)\n", "[screen] nothing queued — ledger already holds 7 rows in exp011_smoke/results.csv; call report(fc) to view standings.\n", " ✓ empty-queue guard returns ledger standings\n", "acd_forge smoke: ALL GREEN — next: phase2_screen()\n" ] } ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_lm_adapter.py — exp_011 Phase 3 (Tier-L)\n", "# Composed micro-aleph addresses conditioning the byte-trigram LM.\n", "#\n", "# Injection seam (source-grounded, aleph_lm.py L474/L610): every head reads\n", "# h = backbone(ids), so conditioning h routes prediction THROUGH the composed\n", "# structure — the employment law, satisfied with zero head surgery:\n", "#\n", "# h = AlephLM.backbone(ids) # (B, S, d)\n", "# h' = h + alpha * W_out( ACD(h) ) # alpha init 0.1 (the\n", "# # simplex-injection precedent)\n", "#\n", "# Tier-L metrics map (documented, units differ from Tier-P):\n", "# ce_bits = bpb (bits per byte, the LM's delivered channel)\n", "# delivered_bits = H0 - bpb (bits ABOVE the measured unigram floor;\n", "# H0 = bias-only probe entropy, so held and delivered\n", "# share one baseline and one scale)\n", "# cum/marginal = staged probes predicting the NEXT BYTE (256-way)\n", "# from cached token addresses (same estimator, same\n", "# class-scaled budget)\n", "#\n", "# Paste order: acd_structures -> acd_probe -> aleph_lm (cells 1-4 of the\n", "# -lm repo, or `from aleph_lm import ...`) -> acd_lm_adapter -> acd_forge.\n", "# ============================================================\n", "from __future__ import annotations\n", "import math, time\n", "from dataclasses import dataclass\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure\n", " from acd_probe import _probe_ce_bits, composition_gauges\n", " from aleph_lm import AlephLM, AlephLMConfig\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver. Pasted Colab cells share ONE namespace and are\n", " not importable modules — so resolve names from globals() first (paste\n", " mode), then fall back to a real import (script/module mode).\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig — the canonical Tier-L recipe (exp_010's 6.75M, sweep-proven)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def tier_l_overrides(**extra) -> Dict:\n", " \"\"\"dim=384 x 4 layers, the exp_010 small backbone. Colab default.\n", " head='byte' keeps the adapter bank-free; hybrid/apmix arms pass\n", " bank_source explicitly (exp_010 convention).\"\"\"\n", " base = dict(dim=384, n_layers=4, n_heads=6, K=64, seq_len=256,\n", " head=\"byte\", batch_size=32, lr=5e-4, amp=False)\n", " base.update(extra)\n", " return base\n", "\n", "\n", "def tier_l_hybrid_overrides(**extra) -> Dict:\n", " \"\"\"apmix_src arms: the exp_010 hybrid recipe, corpus-built bank.\"\"\"\n", " base = tier_l_overrides(head=\"hybrid\", bank_scorer=\"apmix\",\n", " n_pointers=4, bank_source=\"corpus\")\n", " base.update(extra)\n", " return base\n", "\n", "\n", "def smoke_overrides(**extra) -> Dict:\n", " base = dict(dim=64, n_layers=2, n_heads=2, K=8, seq_len=64,\n", " head=\"byte\", batch_size=8, lr=3e-3, amp=False)\n", " base.update(extra)\n", " return base\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# ACDConditioner — token-wise composed address, alpha-gated residual\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class ACDConditioner(nn.Module):\n", " \"\"\"h (B,S,d) -> h + alpha * W_out(ACD(h)). Caches the per-token stage\n", " addresses (subsampled) for the gauge battery. No norm/dropout on the\n", " geometric path (statute).\"\"\"\n", "\n", " def __init__(self, d_model: int, acd_cfg: \"ACDConfig\",\n", " alpha_init: float = 0.1, cache_positions: int = 2048):\n", " super().__init__()\n", " acd_cfg.d_in = d_model\n", " self.acd = ACDStructure(acd_cfg)\n", " self.out = nn.Linear(self.acd.cfg.feature_dim, d_model, bias=False)\n", " torch.nn.init.orthogonal_(self.out.weight)\n", " self.alpha = nn.Parameter(torch.tensor(float(alpha_init)))\n", " self.cache_positions = cache_positions\n", " self.cached_addresses: Optional[Tensor] = None # (N, m, 2K_max)\n", " self.cached_flat_idx: Optional[Tensor] = None # (N,) into B*S\n", "\n", " def forward(self, h: Tensor) -> Tensor:\n", " B, S, d = h.shape\n", " flat = h.reshape(B * S, d)\n", " feats, addrs = self.acd(flat) # (BS,F), (BS,m,2K)\n", " # live caches for coupling modes (grad-carrying; consumed same step)\n", " self._last_feats = feats.reshape(B, S, -1)\n", " self._last_addr = addrs.reshape(B, S, -1) # (B,S,m*2K)\n", " if not self.training:\n", " n = min(self.cache_positions, B * S)\n", " idx = torch.randperm(B * S, device=h.device)[:n]\n", " self.cached_addresses = addrs[idx].detach()\n", " self.cached_flat_idx = idx\n", " return h + self.alpha * self.out(feats).reshape(B, S, d)\n", "\n", "\n", "_ACD_ALEPH_LM_CLS = None\n", "\n", "def acd_aleph_lm_cls():\n", " \"\"\"Lazy subclass factory: pasted cells cannot subclass a base that\n", " hasn't been pasted yet, so ACDAlephLM materializes at FIRST USE,\n", " resolving AlephLM through the shared namespace (or import). Cached;\n", " also published to globals() as ACDAlephLM once built.\"\"\"\n", " global _ACD_ALEPH_LM_CLS\n", " if _ACD_ALEPH_LM_CLS is not None:\n", " return _ACD_ALEPH_LM_CLS\n", " _AlephLM = _ns(\"AlephLM\", \"aleph_lm\")\n", "\n", " class _PmixFromAddr(nn.Module):\n", " \"\"\"Drop-in for W_pmix whose SOURCE is the composed address, not h.\n", " forward(h) ignores h's content (shape only) and reads the\n", " conditioner's live address cache — CE gradients flow through the\n", " pointers into every stage codebook. Head-through-structure.\"\"\"\n", "\n", " def __init__(self, conditioner, addr_width: int, out_width: int):\n", " super().__init__()\n", " self.cond = [conditioner] # list = no param reg\n", " self.lin = nn.Linear(addr_width, out_width, bias=False)\n", " torch.nn.init.orthogonal_(self.lin.weight)\n", "\n", " def forward(self, h: Tensor) -> Tensor:\n", " a = self.cond[0]._last_addr # (B,S,m*2K)\n", " a = a.reshape(*h.shape[:-1], a.shape[-1])\n", " return self.lin(a)\n", "\n", " class ACDAlephLM(_AlephLM):\n", " \"\"\"AlephLM whose backbone output is ACD-conditioned, with a\n", " coupling ladder controlling HOW the head meets the structure:\n", " whisper — h + alpha*W(f) (control; the v1 path)\n", " concat — h' = W_cat([h ; W_f f]), identity-block init\n", " gate — h' = h + sigma(w.[h;f]+b)*W_f f, b init -2\n", " apmix_src — whisper AND the apmix pointer source becomes the\n", " composed address (requires head='hybrid',\n", " bank_scorer='apmix'); the strongest employment.\"\"\"\n", "\n", " def __init__(self, cfg, acd_cfg: \"ACDConfig\",\n", " alpha_init: float = 0.1, coupling: str = \"whisper\",\n", " bank=None):\n", " super().__init__(cfg, bank=bank)\n", " assert coupling in (\"whisper\", \"concat\", \"gate\", \"apmix_src\")\n", " self.coupling = coupling\n", " self.conditioner = ACDConditioner(cfg.dim, acd_cfg, alpha_init)\n", " F_dim = self.conditioner.acd.cfg.feature_dim\n", " d = cfg.dim\n", " if coupling == \"concat\":\n", " self.W_cat = nn.Linear(d + F_dim, d, bias=False)\n", " with torch.no_grad(): # identity-block init:\n", " self.W_cat.weight.zero_() # cold-start == h\n", " self.W_cat.weight[:, :d] = torch.eye(d)\n", " elif coupling == \"gate\":\n", " self.gate_w = nn.Linear(d + F_dim, 1, bias=True)\n", " torch.nn.init.zeros_(self.gate_w.weight)\n", " torch.nn.init.constant_(self.gate_w.bias, -2.0)\n", " self.gate_out = nn.Linear(F_dim, d, bias=False)\n", " torch.nn.init.orthogonal_(self.gate_out.weight)\n", " elif coupling == \"apmix_src\":\n", " assert getattr(cfg, \"head\", \"\") == \"hybrid\" and \\\n", " getattr(cfg, \"bank_scorer\", \"\") == \"apmix\", \\\n", " \"apmix_src needs head='hybrid', bank_scorer='apmix'\"\n", " m = self.conditioner.acd.n_stage_vec\n", " addr_w = 2 * self.conditioner.acd.K_max * m\n", " self.W_pmix = _PmixFromAddr(\n", " self.conditioner, addr_w,\n", " self.W_pmix.out_features\n", " if hasattr(self.W_pmix, \"out_features\")\n", " else self.W_pmix.weight.shape[0])\n", "\n", " def backbone(self, ids: Tensor) -> Tensor:\n", " h = self.conditioner(super().backbone(ids)) # whisper always on\n", " if self.coupling == \"concat\":\n", " f = self.conditioner._last_feats\n", " h = self.W_cat(torch.cat([h, f], dim=-1))\n", " elif self.coupling == \"gate\":\n", " f = self.conditioner._last_feats\n", " g = torch.sigmoid(self.gate_w(torch.cat([h, f], dim=-1)))\n", " h = h + g * self.gate_out(f)\n", " return h\n", "\n", " def acd_param_groups(self, lr: float, geom_lr_mult: float = 0.1):\n", " \"\"\"Optimizer groups per the standing Adam-alignment rule:\n", " codebooks on a slow lane, everything else at base lr.\"\"\"\n", " geom_ids, geom = set(), []\n", " for n, p in self.named_parameters():\n", " if \"codebook\" in n or \"branch_books\" in n:\n", " geom_ids.add(id(p)); geom.append(p)\n", " rest = [p for p in self.parameters() if id(p) not in geom_ids]\n", " return [dict(params=rest, lr=lr),\n", " dict(params=geom, lr=lr * geom_lr_mult, weight_decay=0.0)]\n", "\n", " _ACD_ALEPH_LM_CLS = ACDAlephLM\n", " globals()[\"ACDAlephLM\"] = ACDAlephLM\n", " return ACDAlephLM\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Synthetic stream — order-2 Markov bytes (sandbox smoke; Colab uses\n", "# TrigramStream on WikiText, same API surface)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class MarkovByteStream:\n", " \"\"\"Learnable-structure byte source: 48-symbol alphabet, sparse order-2\n", " transition table. sample(B, S, device) -> ids, targets: (B, S, 3) longs\n", " (next-trigram prediction framing, mirrors TrigramStream).\"\"\"\n", "\n", " def __init__(self, seed: int = 0, n_sym: int = 48, branch: int = 4,\n", " length: int = 200_000):\n", " g = torch.Generator().manual_seed(seed)\n", " import numpy as np\n", " nxt = torch.randint(0, n_sym, (n_sym, n_sym, branch), generator=g)\n", " buf = np.empty(length, dtype=np.uint8)\n", " a = b = 0\n", " for i in range(length):\n", " c = nxt[a, b, torch.randint(0, branch, (1,), generator=g)].item()\n", " buf[i] = 32 + c # printable-ish bytes\n", " a, b = b, c\n", " self.stream = buf # numpy: build_corpus_bank-compatible\n", " self._g = g\n", "\n", " def sample(self, batch: int, seq_len: int, device=\"cpu\"):\n", " need = 3 * (seq_len + 1)\n", " starts = torch.randint(0, len(self.stream) - need - 3,\n", " (batch,), generator=self._g)\n", " starts = (starts // 3) * 3\n", " import numpy as np\n", " rows = torch.from_numpy(\n", " np.stack([self.stream[s:s + need] for s in starts])).long()\n", " tri = rows.view(batch, seq_len + 1, 3)\n", " return tri[:, :-1].to(device), tri[:, 1:].to(device)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier-L gauges — next-byte staged probes on cached token addresses\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@torch.no_grad()\n", "def _collect_lm(model: \"ACDAlephLM\", stream, seq_len: int,\n", " batches: int, batch: int, device):\n", " model.eval()\n", " A, Y = [], []\n", " for _ in range(batches):\n", " ids, tg = stream.sample(batch, seq_len, device)\n", " model.backbone(ids) # fills the cache\n", " addrs = model.conditioner.cached_addresses # (N, m, 2K)\n", " idx = model.conditioner.cached_flat_idx\n", " y = tg[..., 0].reshape(-1)[idx] # next byte 0\n", " A.append(addrs); Y.append(y)\n", " return torch.cat(A), torch.cat(Y)\n", "\n", "\n", "def lm_marginal_bits(model, stream, seq_len: int, device,\n", " batches: int = 4, batch: int = 16,\n", " probe_steps: int = 640, seed: int = 0) -> Dict:\n", " \"\"\"H0 REBASING: the empirical byte distribution is far from uniform\n", " (WikiText unigram entropy ~4.4 bits, not 8), so a probe on ANY input\n", " \"recovers\" ~3.6 bits of pure prior. H0 = bias-only probe (constant\n", " input) measures that floor; marginals and the curve are reported as\n", " BITS ABOVE UNIGRAM. Stage-1 marginal = H0 - H(Y|a_1).\"\"\"\n", " addrs, y = _collect_lm(model, stream, seq_len, batches, batch, device)\n", " N, m, W = addrs.shape\n", " ones = torch.ones(N, 1, device=addrs.device)\n", " H0 = _probe_ce_bits(ones, y, 256, steps=probe_steps, seed=seed - 1)\n", " H0 = min(H0, 8.0)\n", " H_prev = H0\n", " curve, marg, acc = [], [], 0.0\n", " for t in range(m):\n", " prefix = addrs[:, : t + 1].reshape(N, -1)\n", " out = _probe_ce_bits(prefix, y, 256, steps=probe_steps,\n", " seed=seed + t, return_acc=(t == m - 1))\n", " H_t, acc = out if t == m - 1 else (out, acc)\n", " H_t = min(H_t, H_prev)\n", " marg.append(H_prev - H_t)\n", " curve.append(H0 - H_t)\n", " H_prev = H_t\n", " return {\"marginal_bits\": marg, \"cumulative_bits\": curve,\n", " \"H0\": H0, \"probe_acc\": acc, \"addresses\": addrs}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# run_arm_L — one arm, one rung (forge row schema)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "_BANK_CACHE: Dict[tuple, \"Tensor\"] = {}\n", "\n", "def _get_corpus_bank(stream, M: int):\n", " \"\"\"Top-M trigram bank from the stream, cached per (stream, M) —\n", " identical bank across arms of a screen (fairness + speed).\"\"\"\n", " key = (id(stream), M)\n", " if key not in _BANK_CACHE:\n", " build = _ns(\"build_corpus_bank\", \"aleph_lm\")\n", " _BANK_CACHE[key] = build(stream, M)\n", " return _BANK_CACHE[key]\n", "\n", "\n", "def run_arm_L(spec, steps: int, stream, lm_over: Dict,\n", " device: str, probe_steps: int = 640,\n", " eval_every: int = 100) -> Dict[str, object]:\n", " import json as _json\n", " torch.manual_seed(spec.seed)\n", " _Cfg = _ns(\"AlephLMConfig\", \"aleph_lm\")\n", " cfg = _Cfg(**lm_over)\n", " acd = spec.to_acd(d_in=cfg.dim, feature_dim=min(128, 2 * cfg.dim))\n", " bank = None\n", " if getattr(cfg, \"head\", \"byte\") in (\"hybrid\", \"bank\"):\n", " bank = _get_corpus_bank(stream, int(getattr(cfg, \"bank_size\", 4096)))\n", " model = acd_aleph_lm_cls()(\n", " cfg, acd, coupling=getattr(spec, \"coupling\", \"whisper\"),\n", " bank=bank).to(device)\n", " opt = torch.optim.Adam(model.acd_param_groups(lm_over.get(\"lr\", 5e-4)))\n", " t0, killed = time.time(), None\n", " model.train()\n", " for step in range(1, steps + 1):\n", " ids, tg = stream.sample(cfg.batch_size, cfg.seq_len, device)\n", " loss, _aux = model.forward_loss(ids, tg)\n", " opt.zero_grad()\n", " loss.backward()\n", " gn = torch.nn.utils.clip_grad_norm_(model.parameters(),\n", " max(loss.item(), 1.0))\n", " opt.step()\n", " if step % eval_every == 0 and not math.isfinite(loss.item()):\n", " killed = \"NaN/inf loss\"\n", " break\n", " # eval bpb on held batches\n", " model.eval()\n", " with torch.no_grad():\n", " tot, n = 0.0, 0\n", " for _ in range(4):\n", " ids, tg = stream.sample(cfg.batch_size, cfg.seq_len, device)\n", " l, _ = model.forward_loss(ids, tg)\n", " tot += l.item(); n += 1\n", " bpb = (tot / n) / math.log(2) / 3.0 # nats/trigram -> bits/byte\n", " mb = lm_marginal_bits(model, stream, cfg.seq_len, device,\n", " probe_steps=probe_steps, seed=spec.seed)\n", " H0 = mb[\"H0\"]\n", " cg = composition_gauges(model.conditioner.acd,\n", " torch.zeros(1, cfg.dim, device=device)) \\\n", " if False else _addr_gauges(mb[\"addresses\"])\n", " st = model.conditioner.acd.codebook_stats()\n", " return dict(arm_id=spec.arm_id(), op=spec.op, m=spec.m, K=spec.K,\n", " d_addr=spec.d_addr, freeze=spec.freeze, seed=spec.seed,\n", " tree_hard=spec.tree_hard, steps=steps,\n", " params=sum(p.numel() for p in model.parameters()),\n", " acc=round(mb[\"probe_acc\"], 4), ce_bits=round(bpb, 4),\n", " delivered_bits=round(H0 - bpb, 4), # bits above unigram\n", " cum_bits=round(mb[\"cumulative_bits\"][-1], 4),\n", " marginal_bits=_json.dumps(\n", " [round(v, 4) for v in mb[\"marginal_bits\"]]),\n", " redundancy=cg[\"redundancy\"],\n", " cancellation=cg[\"cancellation\"], stage_snr=0.0,\n", " dev_mean=round(sum(s[\"deviation\"] for s in st) / len(st), 4),\n", " rank_mean=round(sum(s[\"eff_rank\"] for s in st) / len(st), 3),\n", " alpha=round(model.conditioner.alpha.item(), 4),\n", " killed=killed or \"\", wall_s=round(time.time() - t0, 1))\n", "\n", "\n", "@torch.no_grad()\n", "def _addr_gauges(addrs: Tensor) -> Dict[str, float]:\n", " \"\"\"redundancy/cancellation on cached (N, m, 2K) token addresses —\n", " same definitions as acd_probe.composition_gauges.\"\"\"\n", " N, m, W = addrs.shape\n", " K = W // 2\n", " if m == 1:\n", " return {\"redundancy\": 0.0, \"cancellation\": 0.0}\n", " flat = F.normalize(addrs, dim=-1)\n", " cos = torch.einsum(\"bmw,bnw->bmn\", flat, flat)\n", " iu = torch.triu_indices(m, m, offset=1)\n", " red = cos[:, iu[0], iu[1]].mean().item()\n", " net = addrs[..., :K] - addrs[..., K:]\n", " agree = (net.sign().sum(dim=1).abs() == m).float()\n", " canc = (((1 - agree) * net.abs().sum(1)).sum(-1)\n", " / net.abs().sum(dim=(1, 2)).clamp_min(1e-9)).mean().item()\n", " return {\"redundancy\": round(red, 4), \"cancellation\": round(canc, 4)}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke — synthetic stream, tiny LM, prod_m4 conditioner\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " try:\n", " ArmSpec = _ns(\"ArmSpec\", \"acd_forge\")\n", " except Exception:\n", " # forge not pasted yet (it comes AFTER the adapter): a minimal shim\n", " # covering exactly the surface run_arm_L touches. Real runs always\n", " # use the forge grammar; the shim exists only for this smoke.\n", " from dataclasses import dataclass as _dc\n", " @_dc\n", " class ArmSpec:\n", " op: str\n", " m: int\n", " K: int = 0\n", " d_addr: int = 4\n", " freeze: str = \"free\"\n", " seed: int = 1234\n", " tree_hard: bool = False\n", " def resolve(self):\n", " if self.K == 0:\n", " self.K = max(2, 256 // (max(self.m, 1) * self.d_addr))\n", " if self.op == \"single\":\n", " self.m = 1\n", " return self\n", " def arm_id(self):\n", " return f\"smoke_{self.op}_m{self.m}_K{self.K}_s{self.seed}\"\n", " def to_acd(self, d_in, feature_dim=64):\n", " return ACDConfig(op=self.op, d_in=d_in, m=self.m, K=self.K,\n", " d_addr=self.d_addr, freeze=self.freeze,\n", " tree_hard=self.tree_hard, seed=self.seed,\n", " feature_dim=feature_dim)\n", " print(\" - ArmSpec shim in use (real grammar arrives with acd_forge)\")\n", " dev = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " stream = MarkovByteStream(seed=3)\n", " ids, tg = stream.sample(4, 32, dev)\n", " assert ids.shape == (4, 32, 3) and tg.shape == (4, 32, 3)\n", " print(f\" ✓ MarkovByteStream: trigram framing {tuple(ids.shape)}\")\n", "\n", " try:\n", " _ns(\"AlephLM\", \"aleph_lm\")\n", " except Exception:\n", " print(\" - aleph-lm cells not in namespace yet: arm smoke DEFERRED \"\n", " \"(paste cells 1-4, then re-run _smoke() — or it validates on \"\n", " \"the first phase3 arm)\")\n", " print(\"acd_lm_adapter smoke: PARTIAL (stream green, arms deferred)\")\n", " return\n", " over = smoke_overrides()\n", " spec = ArmSpec(op=\"prod\", m=4, K=8, seed=1234).resolve()\n", " row = run_arm_L(spec, steps=40, stream=stream, lm_over=over,\n", " device=dev, probe_steps=200, eval_every=20)\n", " assert math.isfinite(row[\"ce_bits\"]) and row[\"ce_bits\"] < 8.0\n", " assert isinstance(row[\"delivered_bits\"], float)\n", " assert -1.0 < row[\"delivered_bits\"] < 8.0 # rebased scale sanity\n", " assert 0.0 <= row[\"acc\"] <= 1.0 # probe top-1\n", " assert isinstance(row[\"alpha\"], float) # injection gate logged\n", " import json as _json\n", " marg = _json.loads(row[\"marginal_bits\"])\n", " assert len(marg) == 4 and all(v >= 0 for v in marg)\n", " print(f\" ✓ run_arm_L: bpb {row['ce_bits']:.3f} delivered \"\n", " f\"{row['delivered_bits']:.3f} held {row['cum_bits']:.3f} \"\n", " f\"{marg} red {row['redundancy']:.2f} \"\n", " f\"dev {row['dev_mean']:+.3f} rank {row['rank_mean']:.2f}\")\n", "\n", " # single-arm control path (m=1 gauges degenerate cleanly)\n", " s1 = ArmSpec(op=\"single\", m=1, K=32, seed=1234).resolve()\n", " r1 = run_arm_L(s1, steps=20, stream=stream, lm_over=over,\n", " device=dev, probe_steps=150, eval_every=20)\n", " assert r1[\"redundancy\"] == 0.0\n", " print(f\" ✓ single control: bpb {r1['ce_bits']:.3f}\")\n", "\n", " # alpha gate is learnable and moved\n", " print(\"acd_lm_adapter smoke: ALL GREEN\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "wh8FR7TWw1Zx", "outputId": "4004b45f-352f-431f-c797-a5d989ab56cf" }, "execution_count": 9, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ MarkovByteStream: trigram framing (4, 32, 3)\n", " ✓ run_arm_L: bpb 5.598 delivered 0.039 held 0.036 [0.0193, 0.0, 0.0004, 0.0161] red 0.16 dev -0.010 rank 3.57\n", " ✓ single control: bpb 5.636\n", "acd_lm_adapter smoke: ALL GREEN\n" ] } ] }, { "cell_type": "code", "source": [ "phase3b_screen()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "6733b18541f647e286c01d09483912e1", "bf7d9ebb6f8f42bab6d2bdd8c54b0af0", "b0da5e31e49849b082a9dd713b35a84f", "7007d889a446427b90e73f7d55f2823d", "985c132d01f34f12aa6fdbfefffe76a8", "f9d8f8d0b0894a2691fbfdb1986e83d7", "6a4ee6077e3843b8aa0a422a808be524", "4fe3c25911d4436698a113969999257e", "fa67416dee644db289200d5cfb3b244a", "ce48f8202ddf47cd9dc682f50cfd2f11", "a241b526154e41e19cf7f86dc5d413c8", "5544fffa9f774b84b55e58e7dc67aed7", "bd3248211f7b4a25aa6bc8ca5d585af3", "b62582c307234c9d91db9fee83c23b62", "d823562bae52442a9539460c934f71d2", "e35a04a752a44c0392c29a863cbaed20", "44c143c321414c33ae99f753d1354c7d", "01caddb991c64455b64b1e33b0c87b41", "97a1dc073fc846d1a97a5abaeeb5c616", "9437f3e441054770921d99c587237508", "697e26c0a0364989ac983ec5e785f2bd", "abc2f44a38f649d4a7d97144306644f3", "e9726a9df5154b8180bdb709ada22c5c", "f92792302ba1456591c24ac7e4ecd89a", "5d57c42ac7a2491b876567559abdc1e3", "d0be94efaf5b4f3fbdaf1df5748dd03f", "4c65959847034e55868634793d494c12", "83814b55220245a3971a5f042e1358f9", "757c3bc75b4f43a6b8eb407d27c3959b", "28d1f1856bc245aaa3f67e2df10ae9ba", "209da5ac5a414b47915ed69b683418f4", "8d2718f3563044f9aaef968a8a611efd", "d03a6a12939047799acd1e2e0f39a268", "a9fb8e7ea07e4a1f9e6973015af74b7f", "3f581a8eded546e6a2a4196bff5e4629", "2517b9b41f1440dba98d167fdd57064b", "8709459dde304673ac0ac86d0921ba9c", "f27d39cfc676444fb756875d3843d1e6", "3c65c222c3e449d8853b9d0b89a4fa0d", "ab30536956cd402a9b4ce26818a0f737", "cb6af780886d44cebb925b6ab9fdfa40", "d445523d9bcb47d1bdbf105bdaa6992a", "bf5b6b452ede4ddd846690f6baeea4f6", "32dc5aad1ffd4ac0a24d4ae2d4ad7253", "9746d2db002e4268abd96ca59377d602", "e75d909801db46cd994f6901b0c88f18", "1a709a29aba745e2b61f6dfa3ce08789", "23248de38bd14e7f9b0600452623f67e", "4a5c5b86e3534ca2928a64a8c47112fc", "f79ec18de4d24bb2b808324977cdcd16", "7425b0cea632486cbcfa0fbebc55f18b", "a2e42797a81244069b8f223645bcaf8c", "e10272655204493eba8ebab078d2c6c7", "036a1029e0574a959c6c1176f15351e4", "8e7a756a4645493e8ac4fdd7d017f1a4", "6f71c712ab2b419fb9e09727f1bae912", "97792cf5f93640628705b3f50c66f25d", "458bb0d5bc254ab6ad2acf061ee9e920", "9a14b826555047a297591de433ffce1c", "80740ea00d8d42c594b08baf11a8f16a", "402d6337f56e4943875d8bfe67516b51", "ad60e58e35d84e509a75246f57e9e921", "7bd66aea01e8413490c9d25845c451a3", "99d4f38b254b481cb2e554852a5c9b02", "e6b7ae5466c44b1ea2ef94ef0b34988b", "31491fe8fa8a4cc8ae10ba7dc34c02f4", "85c335c0e3634b90aab122c9dbfe8bdf", "98a113812d1f40c5b4a7df00f437fc57", "104c0330a7924d6c968bab19d47d821c", "8f33187de10749898eed245e3f29bd2f", "cf1bc2a8be6a4d1d9dbc932ec7ccb74f", "46af79a6fcae48728b6aa74b1bf22f3d", "c9bad91fe7e1472b8423f1de82c87ef0", "e3c97785318b4fd4858afe7749a28974", "f402b8cf12454f6984d2a98fb6161efa", "ad6375b73c554213975e80528318bfa3", "507a01d7449041568ca10f94735eae58", "c35f05059a7d423f9758265cfc2d45e5", "8ef2e29e4e3746bd858d377837659065", "a0303f4db2a745e3bd86b9437cf3b4f0", "927db1fe63c6410294c4cd3eabab5e68", "c4c50f17cd50456e8749134980953dcd", "f18ce9c9c3614872b9c6f58b60320e17", "67d1880b56f4410fa1107c05f30c403c", "8a0b7ed6756047aa99ff1a3742071450", "6fd164f995d943d1a50a3e41f134fdc2", "4c8cdc6dce984019ac5a13e89f9c4f1b", "033bf4e55a0f46f59fc85e41723e0e06" ] }, "id": "sCZ8Zs9wxTNZ", "outputId": "508b8fdc-7cf6-4d20-e205-2ceac06e4619" }, "execution_count": 10, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[forge] 3b: 10 byte-head + 2 hybrid arms\n", "[TrigramStream] loading HF corpus wikitext-103-raw-v1 ...\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "README.md: 0%| | 0.00/10.5k [00:00 AbstractPhil/geolip-aleph-differentiation ✓\n", "\n", "[rung 1] tier=L steps=5000 arms=10 (cached 0, fresh 10)\n", " [1/10] res_m8_K8_gate_free_s1234_3ee6ce held 1.47 ce 2.788 dlv 1.79 [0.4303, 0.3588, 0.203, 0.2402, 0.1145, 0.0732, 0.0506, 0.0] acc 0.403 red 0.09 a 0.128\n", " [2/10] single_m1_K64_free_s5678_800880 held 0.06 ce 2.815 dlv 1.76 [0.0566] acc 0.197 red 0.00 a 0.222\n", " [3/10] sum_m8_K8_free_s5678_ccb02a held 0.99 ce 2.792 dlv 1.85 [0.4559, 0.3058, 0.0564, 0.0, 0.0915, 0.0523, 0.0, 0.0297] acc 0.316 red 0.15 a 0.221\n", " [4/10] res_m8_K8_concat_free_s5678_ce2a0e held 1.58 ce 2.814 dlv 1.85 [0.4745, 0.4482, 0.2507, 0.1154, 0.1339, 0.0508, 0.0549, 0.0478] acc 0.417 red 0.11 a 0.139\n", " [5/10] res_m8_K8_gate_free_s5678_4d541b held 1.47 ce 2.813 dlv 1.74 [0.5484, 0.3624, 0.1315, 0.0803, 0.199, 0.0789, 0.0649, 0.0027] acc 0.416 red 0.13 a 0.138\n", " [6/10] res_m8_K8_free_s5678_0c1ab5 held 1.01 ce 2.797 dlv 1.81 [0.0916, 0.1452, 0.2012, 0.0119, 0.0416, 0.1933, 0.3127, 0.0081] acc 0.325 red 0.14 a 0.155\n", " [7/10] res_m8_K8_free_s1234_9db65b held 0.94 ce 2.849 dlv 1.71 [0.2285, 0.1119, 0.0612, 0.0935, 0.3109, 0.0458, 0.0185, 0.0714] acc 0.317 red 0.13 a 0.157\n", " [8/10] sum_m8_K8_free_s1234_7106f9 held 1.04 ce 2.816 dlv 1.76 [0.2045, 0.0293, 0.0368, 0.4961, 0.0267, 0.0427, 0.1871, 0.0152] acc 0.322 red 0.16 a 0.219\n", " [9/10] res_m8_K8_concat_free_s1234_a379d1 held 1.57 ce 2.769 dlv 1.91 [0.4455, 0.3527, 0.2487, 0.1575, 0.0732, 0.1763, 0.0, 0.1202] acc 0.419 red 0.12 a 0.141\n", " [10/10] single_m1_K64_free_s1234_194318 held 0.08 ce 2.789 dlv 1.84 [0.0798] acc 0.206 red 0.00 a 0.223\n", "[push] exp011H/ -> AbstractPhil/geolip-aleph-differentiation ✓\n", "[TrigramStream] loading HF corpus wikitext-103-raw-v1 ...\n", "[TrigramStream] 100,000,000 bytes = 33,333,333 trigrams\n", "\n", "[rung 0] tier=L steps=1000 arms=2 (cached 0, fresh 2)\n", " [1/2] res_m8_K8_apmix_src_free_s1234_3f7fea held 1.44 ce 3.106 dlv 1.53 [0.5196, 0.2962, 0.2035, 0.1776, 0.0734, 0.1466, 0.0264, 0.0] acc 0.409 red 0.11 a 0.192\n", " [2/2] res_m8_K8_apmix_src_free_s5678_f6a0ea held 1.40 ce 3.045 dlv 1.47 [0.5937, 0.3162, 0.1689, 0.0473, 0.1615, 0.0134, 0.0429, 0.0589] acc 0.409 red 0.12 a 0.200\n", "[push] exp011H/ -> AbstractPhil/geolip-aleph-differentiation ✓\n", "\n", "[rung 1] tier=L steps=5000 arms=1 (cached 0, fresh 1)\n", " [1/1] res_m8_K8_apmix_src_free_s1234_3f7fea held 1.59 ce 2.798 dlv 1.78 [0.562, 0.2782, 0.368, 0.1511, 0.0575, 0.1118, 0.0586, 0.0] acc 0.440 red 0.13 a 0.220\n", "[push] exp011H/ -> AbstractPhil/geolip-aleph-differentiation ✓\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "[{'arm_id': 'res_m8_K8_free_s1234_9db65b',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6881347,\n", " 'acc': 0.2896,\n", " 'ce_bits': 3.2882,\n", " 'delivered_bits': 1.3522,\n", " 'cum_bits': 0.8789,\n", " 'marginal_bits': '[0.0548, 0.1723, 0.2498, 0.195, 0.1186, 0.0791, 0.0092, 0.0]',\n", " 'redundancy': 0.1198,\n", " 'cancellation': 0.9969,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0166,\n", " 'rank_mean': 3.622,\n", " 'alpha': 0.174,\n", " 'killed': '',\n", " 'wall_s': 30.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_concat_free_s1234_a379d1',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 7077955,\n", " 'acc': 0.3745,\n", " 'ce_bits': 3.2498,\n", " 'delivered_bits': 1.2802,\n", " 'cum_bits': 1.2156,\n", " 'marginal_bits': '[0.5191, 0.2459, 0.1532, 0.1614, 0.0498, 0.051, 0.0352, 0.0]',\n", " 'redundancy': 0.1121,\n", " 'cancellation': 0.9934,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0166,\n", " 'rank_mean': 3.623,\n", " 'alpha': 0.1282,\n", " 'killed': '',\n", " 'wall_s': 29.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_gate_free_s1234_3ee6ce',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6931012,\n", " 'acc': 0.3735,\n", " 'ce_bits': 3.2434,\n", " 'delivered_bits': 1.3824,\n", " 'cum_bits': 1.3343,\n", " 'marginal_bits': '[0.6215, 0.1465, 0.206, 0.1522, 0.0804, 0.056, 0.0, 0.0719]',\n", " 'redundancy': 0.0848,\n", " 'cancellation': 0.9962,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.017,\n", " 'rank_mean': 3.624,\n", " 'alpha': 0.1339,\n", " 'killed': '',\n", " 'wall_s': 29.3,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/10 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s1234_7106f9',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6832203,\n", " 'acc': 0.2419,\n", " 'ce_bits': 3.2304,\n", " 'delivered_bits': 1.3783,\n", " 'cum_bits': 0.6028,\n", " 'marginal_bits': '[0.246, 0.0331, 0.0174, 0.1442, 0.0528, 0.0691, 0.0, 0.0402]',\n", " 'redundancy': 0.303,\n", " 'cancellation': 0.9748,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0167,\n", " 'rank_mean': 3.625,\n", " 'alpha': 0.2326,\n", " 'killed': '',\n", " 'wall_s': 29.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6820547,\n", " 'acc': 0.2114,\n", " 'ce_bits': 3.2657,\n", " 'delivered_bits': 1.3217,\n", " 'cum_bits': 0.1457,\n", " 'marginal_bits': '[0.1457]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0068,\n", " 'rank_mean': 3.93,\n", " 'alpha': 0.2348,\n", " 'killed': '',\n", " 'wall_s': 22.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m8_K8_free_s5678_0c1ab5',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6881347,\n", " 'acc': 0.3137,\n", " 'ce_bits': 3.2544,\n", " 'delivered_bits': 1.319,\n", " 'cum_bits': 0.8916,\n", " 'marginal_bits': '[0.1652, 0.0392, 0.0589, 0.3039, 0.0224, 0.2978, 0.0001, 0.004]',\n", " 'redundancy': 0.1226,\n", " 'cancellation': 0.9885,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0013,\n", " 'rank_mean': 3.603,\n", " 'alpha': 0.1714,\n", " 'killed': '',\n", " 'wall_s': 29.0,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_concat_free_s5678_ce2a0e',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 7077955,\n", " 'acc': 0.3828,\n", " 'ce_bits': 3.2531,\n", " 'delivered_bits': 1.4111,\n", " 'cum_bits': 1.3541,\n", " 'marginal_bits': '[0.5579, 0.3046, 0.0717, 0.1767, 0.0789, 0.111, 0.0169, 0.0365]',\n", " 'redundancy': 0.106,\n", " 'cancellation': 0.993,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.002,\n", " 'rank_mean': 3.606,\n", " 'alpha': 0.1267,\n", " 'killed': '',\n", " 'wall_s': 29.1,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_gate_free_s5678_4d541b',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6931012,\n", " 'acc': 0.3667,\n", " 'ce_bits': 3.2576,\n", " 'delivered_bits': 1.3549,\n", " 'cum_bits': 1.3397,\n", " 'marginal_bits': '[0.5212, 0.22, 0.2832, 0.0384, 0.1645, 0.0, 0.0609, 0.0516]',\n", " 'redundancy': 0.1209,\n", " 'cancellation': 0.9968,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0018,\n", " 'rank_mean': 3.605,\n", " 'alpha': 0.1329,\n", " 'killed': '',\n", " 'wall_s': 29.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 6/10 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s5678_ccb02a',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6832203,\n", " 'acc': 0.2864,\n", " 'ce_bits': 3.2371,\n", " 'delivered_bits': 1.3177,\n", " 'cum_bits': 0.7718,\n", " 'marginal_bits': '[0.0702, 0.222, 0.0609, 0.1981, 0.109, 0.021, 0.0906, 0.0]',\n", " 'redundancy': 0.1763,\n", " 'cancellation': 0.9934,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0015,\n", " 'rank_mean': 3.603,\n", " 'alpha': 0.2344,\n", " 'killed': '',\n", " 'wall_s': 29.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'single_m1_K64_free_s5678_800880',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6820547,\n", " 'acc': 0.2268,\n", " 'ce_bits': 3.2623,\n", " 'delivered_bits': 1.3717,\n", " 'cum_bits': 0.2182,\n", " 'marginal_bits': '[0.2182]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0063,\n", " 'rank_mean': 3.977,\n", " 'alpha': 0.232,\n", " 'killed': '',\n", " 'wall_s': 22.7,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'res_m8_K8_gate_free_s1234_3ee6ce',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6931012,\n", " 'acc': 0.4033,\n", " 'ce_bits': 2.7876,\n", " 'delivered_bits': 1.7945,\n", " 'cum_bits': 1.4706,\n", " 'marginal_bits': '[0.4303, 0.3588, 0.203, 0.2402, 0.1145, 0.0732, 0.0506, 0.0]',\n", " 'redundancy': 0.0927,\n", " 'cancellation': 0.9942,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0184,\n", " 'rank_mean': 3.628,\n", " 'alpha': 0.1282,\n", " 'killed': '',\n", " 'wall_s': 138.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_free_s5678_800880',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6820547,\n", " 'acc': 0.1968,\n", " 'ce_bits': 2.8154,\n", " 'delivered_bits': 1.7596,\n", " 'cum_bits': 0.0566,\n", " 'marginal_bits': '[0.0566]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0064,\n", " 'rank_mean': 3.977,\n", " 'alpha': 0.2219,\n", " 'killed': '',\n", " 'wall_s': 111.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s5678_ccb02a',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6832203,\n", " 'acc': 0.3162,\n", " 'ce_bits': 2.7924,\n", " 'delivered_bits': 1.8509,\n", " 'cum_bits': 0.9916,\n", " 'marginal_bits': '[0.4559, 0.3058, 0.0564, 0.0, 0.0915, 0.0523, 0.0, 0.0297]',\n", " 'redundancy': 0.1522,\n", " 'cancellation': 0.9984,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0029,\n", " 'rank_mean': 3.61,\n", " 'alpha': 0.2209,\n", " 'killed': '',\n", " 'wall_s': 138.6,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_concat_free_s5678_ce2a0e',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 7077955,\n", " 'acc': 0.4167,\n", " 'ce_bits': 2.8143,\n", " 'delivered_bits': 1.853,\n", " 'cum_bits': 1.5763,\n", " 'marginal_bits': '[0.4745, 0.4482, 0.2507, 0.1154, 0.1339, 0.0508, 0.0549, 0.0478]',\n", " 'redundancy': 0.1132,\n", " 'cancellation': 0.994,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.004,\n", " 'rank_mean': 3.614,\n", " 'alpha': 0.1393,\n", " 'killed': '',\n", " 'wall_s': 138.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_gate_free_s5678_4d541b',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6931012,\n", " 'acc': 0.416,\n", " 'ce_bits': 2.813,\n", " 'delivered_bits': 1.7355,\n", " 'cum_bits': 1.4681,\n", " 'marginal_bits': '[0.5484, 0.3624, 0.1315, 0.0803, 0.199, 0.0789, 0.0649, 0.0027]',\n", " 'redundancy': 0.1337,\n", " 'cancellation': 0.9876,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0044,\n", " 'rank_mean': 3.616,\n", " 'alpha': 0.138,\n", " 'killed': '',\n", " 'wall_s': 138.1,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_free_s5678_0c1ab5',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6881347,\n", " 'acc': 0.3247,\n", " 'ce_bits': 2.7974,\n", " 'delivered_bits': 1.8092,\n", " 'cum_bits': 1.0057,\n", " 'marginal_bits': '[0.0916, 0.1452, 0.2012, 0.0119, 0.0416, 0.1933, 0.3127, 0.0081]',\n", " 'redundancy': 0.135,\n", " 'cancellation': 0.9823,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0016,\n", " 'rank_mean': 3.605,\n", " 'alpha': 0.1549,\n", " 'killed': '',\n", " 'wall_s': 137.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_free_s1234_9db65b',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6881347,\n", " 'acc': 0.3169,\n", " 'ce_bits': 2.8493,\n", " 'delivered_bits': 1.7072,\n", " 'cum_bits': 0.9417,\n", " 'marginal_bits': '[0.2285, 0.1119, 0.0612, 0.0935, 0.3109, 0.0458, 0.0185, 0.0714]',\n", " 'redundancy': 0.1269,\n", " 'cancellation': 0.9951,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0183,\n", " 'rank_mean': 3.625,\n", " 'alpha': 0.1569,\n", " 'killed': '',\n", " 'wall_s': 137.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'sum_m8_K8_free_s1234_7106f9',\n", " 'op': 'sum',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6832203,\n", " 'acc': 0.3223,\n", " 'ce_bits': 2.8162,\n", " 'delivered_bits': 1.7603,\n", " 'cum_bits': 1.0384,\n", " 'marginal_bits': '[0.2045, 0.0293, 0.0368, 0.4961, 0.0267, 0.0427, 0.1871, 0.0152]',\n", " 'redundancy': 0.161,\n", " 'cancellation': 0.9886,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.018,\n", " 'rank_mean': 3.629,\n", " 'alpha': 0.2191,\n", " 'killed': '',\n", " 'wall_s': 138.4,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_concat_free_s1234_a379d1',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 7077955,\n", " 'acc': 0.4192,\n", " 'ce_bits': 2.7695,\n", " 'delivered_bits': 1.9063,\n", " 'cum_bits': 1.5741,\n", " 'marginal_bits': '[0.4455, 0.3527, 0.2487, 0.1575, 0.0732, 0.1763, 0.0, 0.1202]',\n", " 'redundancy': 0.1192,\n", " 'cancellation': 0.997,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0188,\n", " 'rank_mean': 3.63,\n", " 'alpha': 0.1408,\n", " 'killed': '',\n", " 'wall_s': 137.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_free_s1234_194318',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6820547,\n", " 'acc': 0.2058,\n", " 'ce_bits': 2.7894,\n", " 'delivered_bits': 1.8368,\n", " 'cum_bits': 0.0798,\n", " 'marginal_bits': '[0.0798]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0066,\n", " 'rank_mean': 3.931,\n", " 'alpha': 0.2228,\n", " 'killed': '',\n", " 'wall_s': 111.6,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 10/10 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_apmix_src_free_s1234_3f7fea',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6948424,\n", " 'acc': 0.4087,\n", " 'ce_bits': 3.1055,\n", " 'delivered_bits': 1.5343,\n", " 'cum_bits': 1.4433,\n", " 'marginal_bits': '[0.5196, 0.2962, 0.2035, 0.1776, 0.0734, 0.1466, 0.0264, 0.0]',\n", " 'redundancy': 0.112,\n", " 'cancellation': 0.9952,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0176,\n", " 'rank_mean': 3.625,\n", " 'alpha': 0.1921,\n", " 'killed': '',\n", " 'wall_s': 42.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 1/2 by cum_bits'},\n", " {'arm_id': 'res_m8_K8_apmix_src_free_s5678_f6a0ea',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6948424,\n", " 'acc': 0.4092,\n", " 'ce_bits': 3.0446,\n", " 'delivered_bits': 1.4701,\n", " 'cum_bits': 1.4028,\n", " 'marginal_bits': '[0.5937, 0.3162, 0.1689, 0.0473, 0.1615, 0.0134, 0.0429, 0.0589]',\n", " 'redundancy': 0.115,\n", " 'cancellation': 0.9934,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0021,\n", " 'rank_mean': 3.604,\n", " 'alpha': 0.2003,\n", " 'killed': '',\n", " 'wall_s': 42.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m8_K8_apmix_src_free_s1234_3f7fea',\n", " 'op': 'res',\n", " 'm': 8,\n", " 'K': 8,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6948424,\n", " 'acc': 0.4404,\n", " 'ce_bits': 2.7983,\n", " 'delivered_bits': 1.7837,\n", " 'cum_bits': 1.5872,\n", " 'marginal_bits': '[0.562, 0.2782, 0.368, 0.1511, 0.0575, 0.1118, 0.0586, 0.0]',\n", " 'redundancy': 0.128,\n", " 'cancellation': 0.9967,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0172,\n", " 'rank_mean': 3.622,\n", " 'alpha': 0.2198,\n", " 'killed': '',\n", " 'wall_s': 204.8,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 1/1 by cum_bits'}]" ] }, "metadata": {}, "execution_count": 10 } ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_attention.py — exp_011 Phase 4: composition WHERE INFORMATION IS CREATED\n", "#\n", "# The sterility law (Phase 3b) proved a post-backbone conditioner cannot add\n", "# bits: I(Y;h) >= I(Y;f(h)). So the composition moves INTO the routing: the\n", "# composed micro-address becomes the attention feature map itself, per layer.\n", "#\n", "# The hub equations are width-agnostic (pure GEMMs over p± of width K), so\n", "# concatenated stage addresses compute an ADDITIVE KERNEL exactly:\n", "# k(q,x) = sum_i phi_i(q) . phi_i(x), phi_i = stage-i signed address.\n", "# Budget rule K_i = K_total/m matches codebook floats AND feature width at\n", "# once — every operator emits width K_total (=64 default), fully fair.\n", "#\n", "# Operators (kernel-valid subset of the ACD taxonomy):\n", "# single — one stage, K=K_total: EXACTLY the stock AlephRoutedAttention\n", "# (parity-gated in the smoke; the control)\n", "# sum — m independent codebooks over the SAME x_hat, addresses\n", "# CONCATENATED: width-fair, codebook-fair, conditioning = NONE\n", "# (the unconditioned-redundancy control, kernel-valid)\n", "# prod — per-stage projections to m INDEPENDENT Da-spheres (the learned\n", "# projection IS the subspace split): conjunctive-by-construction\n", "# res — sequential on the Da-sphere: stage t addresses the renormalized\n", "# residual after soft-reconstruction subtraction\n", "#\n", "# Paste order: aleph cells 1-4 -> acd_structures -> acd_probe ->\n", "# acd_attention -> acd_lm_adapter -> acd_forge.\n", "# ============================================================\n", "from __future__ import annotations\n", "import math\n", "from dataclasses import dataclass\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from aleph_routed_attention import (AlephRoutedAttention,\n", " AlephAttentionConfig)\n", " from acd_structures import acd_statute, _acd_max_spread_points\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver (shared-namespace law): globals first, import second.\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "ROUTE_OPS = (\"single\", \"sum\", \"prod\", \"res\")\n", "\n", "@dataclass\n", "class RouteComposeConfig:\n", " op: str = \"single\"\n", " m: int = 1\n", " K_total: int = 64 # feature width AND codebook-float budget\n", " freeze: str = \"free\" # 'free' | 'spread'\n", " seed: int = 1234\n", "\n", " def __post_init__(self):\n", " assert self.op in ROUTE_OPS\n", " if self.op == \"single\":\n", " self.m = 1\n", " assert self.K_total % self.m == 0, \"K_total must divide by m\"\n", "\n", " @property\n", " def K_stage(self) -> int:\n", " return self.K_total // self.m\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# ACDRoutedAttention — surgical subclass: only the address pathway changes\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def acd_routed_attention_cls():\n", " \"\"\"Lazy factory (top-level subclassing of an unpasted base is illegal\n", " in the shared namespace — standing law).\"\"\"\n", " if \"_ACD_RA_CLS\" in globals() and globals()[\"_ACD_RA_CLS\"] is not None:\n", " return globals()[\"_ACD_RA_CLS\"]\n", " _ARA = _ns(\"AlephRoutedAttention\", \"aleph_routed_attention\")\n", " _statute = _ns(\"acd_statute\", \"acd_structures\")\n", " _spread = _ns(\"_acd_max_spread_points\", \"acd_structures\")\n", "\n", " class ACDRoutedAttention(_ARA):\n", " \"\"\"Stock hub attention with a COMPOSED address. Hub math, streaming\n", " state, masking, chunking: all inherited untouched (width-blind).\n", " For op='prod' the q/k address projections widen to H*m*Da so each\n", " stage owns an independent learned Da-sphere.\"\"\"\n", "\n", " def __init__(self, cfg: \"AlephAttentionConfig\",\n", " rc: RouteComposeConfig):\n", " assert cfg.mode == \"hub\", \"composed routing: hub mode v1\"\n", " # stock init builds K=cfg.K codebook + H*Da projections; we\n", " # rebuild what composition changes afterwards.\n", " super().__init__(cfg)\n", " self.rc = rc\n", " m, Ks, Da = rc.m, rc.K_stage, self.Da\n", " g = torch.Generator().manual_seed(rc.seed)\n", " if rc.freeze == \"spread\":\n", " books = torch.stack([_spread(Ks, Da, seed=rc.seed + i)\n", " for i in range(m)])\n", " else:\n", " books = F.normalize(torch.randn(m, Ks, Da, generator=g),\n", " dim=-1)\n", " del self.codebook # replace stock (K,Da)\n", " self.stage_books = nn.Parameter(\n", " books, requires_grad=(rc.freeze == \"free\"))\n", " if rc.op == \"prod\": # widen to per-stage spheres\n", " d = cfg.dim\n", " self.q_addr = nn.Linear(d, self.H * m * Da, bias=False)\n", " nn.init.orthogonal_(self.q_addr.weight)\n", " if cfg.tied_address:\n", " self.k_addr = self.q_addr\n", " else:\n", " self.k_addr = nn.Linear(d, self.H * m * Da, bias=False)\n", " nn.init.orthogonal_(self.k_addr.weight)\n", " if rc.op == \"res\": # Da-space soft decoders\n", " self.res_dec = nn.Parameter(\n", " F.normalize(torch.randn(m, 2 * Ks, Da, generator=g),\n", " dim=-1) * 0.1)\n", " # eval-mode address cache (probe contract, conditioner-style)\n", " self.cache_positions = 2048\n", " self.cached_addresses: Optional[Tensor] = None\n", " self.cached_flat_idx: Optional[Tensor] = None\n", "\n", " # ---- projections ------------------------------------------------\n", " def _split_addr(self, t: Tensor, B: int, S: int) -> Tensor:\n", " if self.rc.op == \"prod\":\n", " t = t.view(B, S, self.H, self.rc.m, self.Da).transpose(1, 2)\n", " return F.normalize(t, dim=-1) # (B,H,S,m,Da) per-stage sphere\n", " return super()._split_addr(t, B, S) # (B,H,S,Da)\n", "\n", " # ---- the composed address ---------------------------------------\n", " def _stage_address(self, x_hat: Tensor, i: int\n", " ) -> Tuple[Tensor, Tensor]:\n", " A = F.normalize(self.stage_books[i], dim=-1) # (Ks, Da)\n", " u = (x_hat @ A.t()) * (1.0 / self.tau)\n", " mm = u.abs().amax(dim=-1, keepdim=True)\n", " ep, en = torch.exp(u - mm), torch.exp(-u - mm)\n", " Z = (ep + en).sum(dim=-1, keepdim=True)\n", " return ep / Z, en / Z\n", "\n", " def _address(self, x_hat: Tensor) -> Tuple[Tensor, Tensor]:\n", " rc = self.rc\n", " ps, ns_ = [], []\n", " if rc.op == \"single\":\n", " p, n = self._stage_address(x_hat, 0)\n", " ps.append(p); ns_.append(n)\n", " elif rc.op == \"prod\": # x_hat: (B,H,S,m,Da)\n", " for i in range(rc.m):\n", " p, n = self._stage_address(x_hat[..., i, :], i)\n", " ps.append(p); ns_.append(n)\n", " elif rc.op == \"sum\": # same input, m books\n", " for i in range(rc.m):\n", " p, n = self._stage_address(x_hat, i)\n", " ps.append(p); ns_.append(n)\n", " else: # res: sphere residual chain\n", " r = x_hat\n", " for i in range(rc.m):\n", " p, n = self._stage_address(r, i)\n", " ps.append(p); ns_.append(n)\n", " recon = torch.cat([p, n], -1) @ self.res_dec[i]\n", " r = F.normalize(r - recon, dim=-1)\n", " P, N = torch.cat(ps, -1), torch.cat(ns_, -1) # width K_total\n", " if not self.training and P.dim() == 4:\n", " B, H, S, W = P.shape\n", " a = torch.cat([P[:, 0], N[:, 0]], -1).reshape(B * S, 2 * W)\n", " n_keep = min(self.cache_positions, B * S)\n", " idx = torch.randperm(B * S, device=P.device)[:n_keep]\n", " Ks = self.rc.K_stage\n", " st = torch.stack([torch.cat([a[:, i*Ks:(i+1)*Ks],\n", " a[:, W+i*Ks:W+(i+1)*Ks]], -1)\n", " for i in range(self.rc.m)], 1) # (BS,m,2Ks)\n", " self.cached_addresses = st[idx].detach()\n", " self.cached_flat_idx = idx\n", " return P, N\n", "\n", " def _confidence(self, pq_p: Tensor, pq_m: Tensor) -> Tensor:\n", " A = F.normalize(self.stage_books, dim=-1).reshape(-1, self.Da)\n", " return ((pq_p - pq_m) @ A).norm(dim=-1)\n", "\n", " def stage_codebook_stats(self) -> List[Dict]:\n", " return [_statute(self.stage_books[i].detach())\n", " for i in range(self.rc.m)]\n", "\n", " globals()[\"_ACD_RA_CLS\"] = ACDRoutedAttention\n", " return ACDRoutedAttention\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# LM surgery — swap every layer's attention for the composed variant\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def compose_routing(model, rc: RouteComposeConfig):\n", " \"\"\"Replace each layer's stock AlephRoutedAttention with the composed\n", " class (fresh init — arms train from scratch). Returns the model.\"\"\"\n", " _Cfg = _ns(\"AlephAttentionConfig\", \"aleph_routed_attention\")\n", " CLS = acd_routed_attention_cls()\n", " for li, layer in enumerate(model.layers):\n", " old = layer[\"attn\"]\n", " c = old.cfg\n", " rc_i = RouteComposeConfig(op=rc.op, m=rc.m, K_total=rc.K_total,\n", " freeze=rc.freeze, seed=rc.seed + 101 * li)\n", " layer[\"attn\"] = CLS(c, rc_i).to(next(old.parameters()).device)\n", " model._route_compose = rc\n", " return model\n", "\n", "\n", "def routing_codebook_stats(model) -> List[Dict]:\n", " out = []\n", " for layer in model.layers:\n", " out += layer[\"attn\"].stage_codebook_stats()\n", " return out\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke — parity crown + all ops fwd/bwd + chunk parity\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " torch.manual_seed(0)\n", " try:\n", " _ARA = _ns(\"AlephRoutedAttention\", \"aleph_routed_attention\")\n", " except Exception:\n", " print(\" - aleph cells not in namespace yet: attention smoke \"\n", " \"DEFERRED (paste cells 1-4, then re-run _smoke()).\")\n", " print(\"acd_attention smoke: PARTIAL (deferred)\")\n", " return\n", " _Cfg = _ns(\"AlephAttentionConfig\", \"aleph_routed_attention\")\n", " CLS = acd_routed_attention_cls()\n", " B, S, d = 2, 48, 64\n", " cfg = _Cfg(dim=d, num_heads=2, mode=\"hub\", K=16, D_addr=4,\n", " tau=0.1, causal=True)\n", " x = torch.randn(B, S, d)\n", "\n", " # ── PARITY CROWN: op='single' must equal stock bit-for-bit ──\n", " stock = _ARA(cfg).eval()\n", " single = CLS(cfg, RouteComposeConfig(op=\"single\", m=1,\n", " K_total=16, seed=7)).eval()\n", " sd = {k: v for k, v in stock.state_dict().items() if k != \"codebook\"}\n", " single.load_state_dict(sd, strict=False)\n", " with torch.no_grad():\n", " single.stage_books.copy_(stock.codebook.unsqueeze(0))\n", " y0, y1 = stock(x), single(x)\n", " assert torch.allclose(y0, y1, atol=1e-5), \\\n", " (y0 - y1).abs().max().item()\n", " print(\" ✓ PARITY: composed(single) ≡ stock AlephRoutedAttention \"\n", " f\"(max Δ {(y0-y1).abs().max().item():.2e})\")\n", "\n", " # ── every op: fwd/bwd finite, width fair, stats live ──\n", " for op, m in ((\"sum\", 4), (\"prod\", 4), (\"res\", 4)):\n", " rc = RouteComposeConfig(op=op, m=m, K_total=16, seed=7)\n", " att = CLS(cfg, rc)\n", " y = att(x)\n", " loss = y.square().mean()\n", " loss.backward()\n", " g = [p.grad for p in att.parameters()\n", " if p.requires_grad and p.grad is not None]\n", " assert y.shape == (B, S, d) and torch.isfinite(y).all()\n", " assert g and all(torch.isfinite(t).all() for t in g)\n", " st = att.stage_codebook_stats()\n", " assert len(st) == m and all(\"eff_rank_ceiling\" in s for s in st)\n", " att.eval(); _ = att(x)\n", " assert att.cached_addresses is not None \\\n", " and att.cached_addresses.shape[1] == m\n", " print(f\" ✓ {op:5s} m={m} K_stage={rc.K_stage} fwd/bwd + \"\n", " f\"cache ({tuple(att.cached_addresses.shape)}) \"\n", " f\"dev0 {st[0]['deviation']:+.3f}\")\n", "\n", " # ── chunk-vs-full parity survives the width change (prod) ──\n", " cfg_nc = _Cfg(dim=d, num_heads=2, mode=\"hub\", K=16, D_addr=4,\n", " tau=0.1, causal=False)\n", " att = CLS(cfg_nc, RouteComposeConfig(op=\"prod\", m=4, K_total=16,\n", " seed=7)).eval()\n", " with torch.no_grad():\n", " qh = att._split_addr(att.q_addr(x), B, S)\n", " kh = att._split_addr(att.k_addr(x), B, S)\n", " v = att.v_proj(x).view(B, S, att.H, att.hd).transpose(1, 2)\n", " pq = att._address(qh); pk = att._address(kh)\n", " full = att._hub_full(*pq, *pk, v)\n", " causal, _ = att._hub_causal(*pq, *pk, v)\n", " prefix = att._hub_full(pq[0][:, :, -1:], pq[1][:, :, -1:],\n", " pk[0], pk[1], v)\n", " assert torch.allclose(causal[:, :, -1], prefix[:, :, -1], atol=1e-4)\n", " print(\" ✓ chunked-causal ≡ full-prefix at the last position \"\n", " \"(streaming exactness inherited)\")\n", " print(\"acd_attention smoke: ALL GREEN\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " # Notebook cells execute as __main__: smoke fires on paste (convention).\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6FBWP3Of9YQh", "outputId": "fe9e79d8-cd46-4096-9e48-21a677c2b8da" }, "execution_count": 11, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ PARITY: composed(single) ≡ stock AlephRoutedAttention (max Δ 0.00e+00)\n", " ✓ sum m=4 K_stage=4 fwd/bwd + cache ((96, 4, 8)) dev0 +0.031\n", " ✓ prod m=4 K_stage=4 fwd/bwd + cache ((96, 4, 8)) dev0 +0.031\n", " ✓ res m=4 K_stage=4 fwd/bwd + cache ((96, 4, 8)) dev0 +0.031\n", " ✓ chunked-causal ≡ full-prefix at the last position (streaming exactness inherited)\n", "acd_attention smoke: ALL GREEN\n" ] } ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_lm_adapter.py — exp_011 Phase 3 (Tier-L)\n", "# Composed micro-aleph addresses conditioning the byte-trigram LM.\n", "#\n", "# Injection seam (source-grounded, aleph_lm.py L474/L610): every head reads\n", "# h = backbone(ids), so conditioning h routes prediction THROUGH the composed\n", "# structure — the employment law, satisfied with zero head surgery:\n", "#\n", "# h = AlephLM.backbone(ids) # (B, S, d)\n", "# h' = h + alpha * W_out( ACD(h) ) # alpha init 0.1 (the\n", "# # simplex-injection precedent)\n", "#\n", "# Tier-L metrics map (documented, units differ from Tier-P):\n", "# ce_bits = bpb (bits per byte, the LM's delivered channel)\n", "# delivered_bits = H0 - bpb (bits ABOVE the measured unigram floor;\n", "# H0 = bias-only probe entropy, so held and delivered\n", "# share one baseline and one scale)\n", "# cum/marginal = staged probes predicting the NEXT BYTE (256-way)\n", "# from cached token addresses (same estimator, same\n", "# class-scaled budget)\n", "#\n", "# Paste order: acd_structures -> acd_probe -> aleph_lm (cells 1-4 of the\n", "# -lm repo, or `from aleph_lm import ...`) -> acd_lm_adapter -> acd_forge.\n", "# ============================================================\n", "from __future__ import annotations\n", "import math, time\n", "from dataclasses import dataclass\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "from torch import Tensor\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure\n", " from acd_probe import _probe_ce_bits, composition_gauges\n", " from aleph_lm import AlephLM, AlephLMConfig\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver. Pasted Colab cells share ONE namespace and are\n", " not importable modules — so resolve names from globals() first (paste\n", " mode), then fall back to a real import (script/module mode).\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig — the canonical Tier-L recipe (exp_010's 6.75M, sweep-proven)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def tier_l_overrides(**extra) -> Dict:\n", " \"\"\"dim=384 x 4 layers, the exp_010 small backbone. Colab default.\n", " head='byte' keeps the adapter bank-free; hybrid/apmix arms pass\n", " bank_source explicitly (exp_010 convention).\"\"\"\n", " base = dict(dim=384, n_layers=4, n_heads=6, K=64, seq_len=256,\n", " head=\"byte\", batch_size=32, lr=5e-4, amp=False)\n", " base.update(extra)\n", " return base\n", "\n", "\n", "def tier_l_hybrid_overrides(**extra) -> Dict:\n", " \"\"\"apmix_src arms: the exp_010 hybrid recipe, corpus-built bank.\"\"\"\n", " base = tier_l_overrides(head=\"hybrid\", bank_scorer=\"apmix\",\n", " n_pointers=4, bank_source=\"corpus\")\n", " base.update(extra)\n", " return base\n", "\n", "\n", "def smoke_overrides(**extra) -> Dict:\n", " base = dict(dim=64, n_layers=2, n_heads=2, K=8, seq_len=64,\n", " head=\"byte\", batch_size=8, lr=3e-3, amp=False)\n", " base.update(extra)\n", " return base\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# ACDConditioner — token-wise composed address, alpha-gated residual\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class ACDConditioner(nn.Module):\n", " \"\"\"h (B,S,d) -> h + alpha * W_out(ACD(h)). Caches the per-token stage\n", " addresses (subsampled) for the gauge battery. No norm/dropout on the\n", " geometric path (statute).\"\"\"\n", "\n", " def __init__(self, d_model: int, acd_cfg: \"ACDConfig\",\n", " alpha_init: float = 0.1, cache_positions: int = 2048):\n", " super().__init__()\n", " acd_cfg.d_in = d_model\n", " self.acd = ACDStructure(acd_cfg)\n", " self.out = nn.Linear(self.acd.cfg.feature_dim, d_model, bias=False)\n", " torch.nn.init.orthogonal_(self.out.weight)\n", " self.alpha = nn.Parameter(torch.tensor(float(alpha_init)))\n", " self.cache_positions = cache_positions\n", " self.cached_addresses: Optional[Tensor] = None # (N, m, 2K_max)\n", " self.cached_flat_idx: Optional[Tensor] = None # (N,) into B*S\n", "\n", " def forward(self, h: Tensor) -> Tensor:\n", " B, S, d = h.shape\n", " flat = h.reshape(B * S, d)\n", " feats, addrs = self.acd(flat) # (BS,F), (BS,m,2K)\n", " # live caches for coupling modes (grad-carrying; consumed same step)\n", " self._last_feats = feats.reshape(B, S, -1)\n", " self._last_addr = addrs.reshape(B, S, -1) # (B,S,m*2K)\n", " if not self.training:\n", " n = min(self.cache_positions, B * S)\n", " idx = torch.randperm(B * S, device=h.device)[:n]\n", " self.cached_addresses = addrs[idx].detach()\n", " self.cached_flat_idx = idx\n", " return h + self.alpha * self.out(feats).reshape(B, S, d)\n", "\n", "\n", "_ACD_ALEPH_LM_CLS = None\n", "\n", "def acd_aleph_lm_cls():\n", " \"\"\"Lazy subclass factory: pasted cells cannot subclass a base that\n", " hasn't been pasted yet, so ACDAlephLM materializes at FIRST USE,\n", " resolving AlephLM through the shared namespace (or import). Cached;\n", " also published to globals() as ACDAlephLM once built.\"\"\"\n", " global _ACD_ALEPH_LM_CLS\n", " if _ACD_ALEPH_LM_CLS is not None:\n", " return _ACD_ALEPH_LM_CLS\n", " _AlephLM = _ns(\"AlephLM\", \"aleph_lm\")\n", "\n", " class _PmixFromAddr(nn.Module):\n", " \"\"\"Drop-in for W_pmix whose SOURCE is the composed address, not h.\n", " forward(h) ignores h's content (shape only) and reads the\n", " conditioner's live address cache — CE gradients flow through the\n", " pointers into every stage codebook. Head-through-structure.\"\"\"\n", "\n", " def __init__(self, conditioner, addr_width: int, out_width: int):\n", " super().__init__()\n", " self.cond = [conditioner] # list = no param reg\n", " self.lin = nn.Linear(addr_width, out_width, bias=False)\n", " torch.nn.init.orthogonal_(self.lin.weight)\n", "\n", " def forward(self, h: Tensor) -> Tensor:\n", " a = self.cond[0]._last_addr # (B,S,m*2K)\n", " a = a.reshape(*h.shape[:-1], a.shape[-1])\n", " return self.lin(a)\n", "\n", " class ACDAlephLM(_AlephLM):\n", " \"\"\"AlephLM whose backbone output is ACD-conditioned, with a\n", " coupling ladder controlling HOW the head meets the structure:\n", " whisper — h + alpha*W(f) (control; the v1 path)\n", " concat — h' = W_cat([h ; W_f f]), identity-block init\n", " gate — h' = h + sigma(w.[h;f]+b)*W_f f, b init -2\n", " apmix_src — whisper AND the apmix pointer source becomes the\n", " composed address (requires head='hybrid',\n", " bank_scorer='apmix'); the strongest employment.\"\"\"\n", "\n", " def __init__(self, cfg, acd_cfg: \"ACDConfig\",\n", " alpha_init: float = 0.1, coupling: str = \"whisper\",\n", " bank=None):\n", " super().__init__(cfg, bank=bank)\n", " assert coupling in (\"whisper\", \"concat\", \"gate\", \"apmix_src\")\n", " self.coupling = coupling\n", " self.conditioner = ACDConditioner(cfg.dim, acd_cfg, alpha_init)\n", " F_dim = self.conditioner.acd.cfg.feature_dim\n", " d = cfg.dim\n", " if coupling == \"concat\":\n", " self.W_cat = nn.Linear(d + F_dim, d, bias=False)\n", " with torch.no_grad(): # identity-block init:\n", " self.W_cat.weight.zero_() # cold-start == h\n", " self.W_cat.weight[:, :d] = torch.eye(d)\n", " elif coupling == \"gate\":\n", " self.gate_w = nn.Linear(d + F_dim, 1, bias=True)\n", " torch.nn.init.zeros_(self.gate_w.weight)\n", " torch.nn.init.constant_(self.gate_w.bias, -2.0)\n", " self.gate_out = nn.Linear(F_dim, d, bias=False)\n", " torch.nn.init.orthogonal_(self.gate_out.weight)\n", " elif coupling == \"apmix_src\":\n", " assert getattr(cfg, \"head\", \"\") == \"hybrid\" and \\\n", " getattr(cfg, \"bank_scorer\", \"\") == \"apmix\", \\\n", " \"apmix_src needs head='hybrid', bank_scorer='apmix'\"\n", " m = self.conditioner.acd.n_stage_vec\n", " addr_w = 2 * self.conditioner.acd.K_max * m\n", " self.W_pmix = _PmixFromAddr(\n", " self.conditioner, addr_w,\n", " self.W_pmix.out_features\n", " if hasattr(self.W_pmix, \"out_features\")\n", " else self.W_pmix.weight.shape[0])\n", "\n", " def backbone(self, ids: Tensor) -> Tensor:\n", " h = self.conditioner(super().backbone(ids)) # whisper always on\n", " if self.coupling == \"concat\":\n", " f = self.conditioner._last_feats\n", " h = self.W_cat(torch.cat([h, f], dim=-1))\n", " elif self.coupling == \"gate\":\n", " f = self.conditioner._last_feats\n", " g = torch.sigmoid(self.gate_w(torch.cat([h, f], dim=-1)))\n", " h = h + g * self.gate_out(f)\n", " return h\n", "\n", " def acd_param_groups(self, lr: float, geom_lr_mult: float = 0.1):\n", " \"\"\"Optimizer groups per the standing Adam-alignment rule:\n", " codebooks on a slow lane, everything else at base lr.\"\"\"\n", " geom_ids, geom = set(), []\n", " for n, p in self.named_parameters():\n", " if \"codebook\" in n or \"branch_books\" in n:\n", " geom_ids.add(id(p)); geom.append(p)\n", " rest = [p for p in self.parameters() if id(p) not in geom_ids]\n", " return [dict(params=rest, lr=lr),\n", " dict(params=geom, lr=lr * geom_lr_mult, weight_decay=0.0)]\n", "\n", " _ACD_ALEPH_LM_CLS = ACDAlephLM\n", " globals()[\"ACDAlephLM\"] = ACDAlephLM\n", " return ACDAlephLM\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Synthetic stream — order-2 Markov bytes (sandbox smoke; Colab uses\n", "# TrigramStream on WikiText, same API surface)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "class MarkovByteStream:\n", " \"\"\"Learnable-structure byte source: 48-symbol alphabet, sparse order-2\n", " transition table. sample(B, S, device) -> ids, targets: (B, S, 3) longs\n", " (next-trigram prediction framing, mirrors TrigramStream).\"\"\"\n", "\n", " def __init__(self, seed: int = 0, n_sym: int = 48, branch: int = 4,\n", " length: int = 200_000):\n", " g = torch.Generator().manual_seed(seed)\n", " import numpy as np\n", " nxt = torch.randint(0, n_sym, (n_sym, n_sym, branch), generator=g)\n", " buf = np.empty(length, dtype=np.uint8)\n", " a = b = 0\n", " for i in range(length):\n", " c = nxt[a, b, torch.randint(0, branch, (1,), generator=g)].item()\n", " buf[i] = 32 + c # printable-ish bytes\n", " a, b = b, c\n", " self.stream = buf # numpy: build_corpus_bank-compatible\n", " self._g = g\n", "\n", " def sample(self, batch: int, seq_len: int, device=\"cpu\"):\n", " need = 3 * (seq_len + 1)\n", " starts = torch.randint(0, len(self.stream) - need - 3,\n", " (batch,), generator=self._g)\n", " starts = (starts // 3) * 3\n", " import numpy as np\n", " rows = torch.from_numpy(\n", " np.stack([self.stream[s:s + need] for s in starts])).long()\n", " tri = rows.view(batch, seq_len + 1, 3)\n", " return tri[:, :-1].to(device), tri[:, 1:].to(device)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier-L gauges — next-byte staged probes on cached token addresses\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _addr_source(model):\n", " \"\"\"Where do probe addresses live? Conditioned models: the conditioner.\n", " Routed models (Phase 4): the LAST layer's composed attention.\"\"\"\n", " if hasattr(model, \"conditioner\"):\n", " return model.conditioner\n", " return model.layers[-1][\"attn\"]\n", "\n", "\n", "@torch.no_grad()\n", "def _collect_lm(model, stream, seq_len: int,\n", " batches: int, batch: int, device):\n", " model.eval()\n", " src_mod = _addr_source(model)\n", " A, Y = [], []\n", " for _ in range(batches):\n", " ids, tg = stream.sample(batch, seq_len, device)\n", " model.backbone(ids) # fills the cache\n", " addrs = src_mod.cached_addresses # (N, m, 2K)\n", " idx = src_mod.cached_flat_idx\n", " y = tg[..., 0].reshape(-1)[idx] # next byte 0\n", " A.append(addrs); Y.append(y)\n", " return torch.cat(A), torch.cat(Y)\n", "\n", "\n", "def lm_marginal_bits(model, stream, seq_len: int, device,\n", " batches: int = 4, batch: int = 16,\n", " probe_steps: int = 640, seed: int = 0) -> Dict:\n", " \"\"\"H0 REBASING: the empirical byte distribution is far from uniform\n", " (WikiText unigram entropy ~4.4 bits, not 8), so a probe on ANY input\n", " \"recovers\" ~3.6 bits of pure prior. H0 = bias-only probe (constant\n", " input) measures that floor; marginals and the curve are reported as\n", " BITS ABOVE UNIGRAM. Stage-1 marginal = H0 - H(Y|a_1).\"\"\"\n", " addrs, y = _collect_lm(model, stream, seq_len, batches, batch, device)\n", " N, m, W = addrs.shape\n", " ones = torch.ones(N, 1, device=addrs.device)\n", " H0 = _probe_ce_bits(ones, y, 256, steps=probe_steps, seed=seed - 1)\n", " H0 = min(H0, 8.0)\n", " H_prev = H0\n", " curve, marg, acc = [], [], 0.0\n", " for t in range(m):\n", " prefix = addrs[:, : t + 1].reshape(N, -1)\n", " out = _probe_ce_bits(prefix, y, 256, steps=probe_steps,\n", " seed=seed + t, return_acc=(t == m - 1))\n", " H_t, acc = out if t == m - 1 else (out, acc)\n", " H_t = min(H_t, H_prev)\n", " marg.append(H_prev - H_t)\n", " curve.append(H0 - H_t)\n", " H_prev = H_t\n", " return {\"marginal_bits\": marg, \"cumulative_bits\": curve,\n", " \"H0\": H0, \"probe_acc\": acc, \"addresses\": addrs}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# run_arm_L — one arm, one rung (forge row schema)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "_BANK_CACHE: Dict[tuple, \"Tensor\"] = {}\n", "\n", "def _get_corpus_bank(stream, M: int):\n", " \"\"\"Top-M trigram bank from the stream, cached per (stream, M) —\n", " identical bank across arms of a screen (fairness + speed).\"\"\"\n", " key = (id(stream), M)\n", " if key not in _BANK_CACHE:\n", " build = _ns(\"build_corpus_bank\", \"aleph_lm\")\n", " _BANK_CACHE[key] = build(stream, M)\n", " return _BANK_CACHE[key]\n", "\n", "\n", "def run_arm_L(spec, steps: int, stream, lm_over: Dict,\n", " device: str, probe_steps: int = 640,\n", " eval_every: int = 100) -> Dict[str, object]:\n", " import json as _json\n", " torch.manual_seed(spec.seed)\n", " _Cfg = _ns(\"AlephLMConfig\", \"aleph_lm\")\n", " cfg = _Cfg(**lm_over)\n", " acd = spec.to_acd(d_in=cfg.dim, feature_dim=min(128, 2 * cfg.dim))\n", " bank = None\n", " if getattr(cfg, \"head\", \"byte\") in (\"hybrid\", \"bank\"):\n", " bank = _get_corpus_bank(stream, int(getattr(cfg, \"bank_size\", 4096)))\n", " if getattr(spec, \"coupling\", \"whisper\") == \"routed\":\n", " # Phase 4: NO conditioner — composition lives inside the layers.\n", " _LM = _ns(\"AlephLM\", \"aleph_lm\")\n", " RC = _ns(\"RouteComposeConfig\", \"acd_attention\")\n", " compose = _ns(\"compose_routing\", \"acd_attention\")\n", " model = _LM(cfg, bank=bank) if bank is not None else _LM(cfg)\n", " model = compose(model, RC(op=spec.op, m=spec.m,\n", " K_total=spec.K or 64,\n", " freeze=spec.freeze,\n", " seed=spec.seed)).to(device)\n", " else:\n", " model = acd_aleph_lm_cls()(\n", " cfg, acd, coupling=getattr(spec, \"coupling\", \"whisper\"),\n", " bank=bank).to(device)\n", " if hasattr(model, \"acd_param_groups\"):\n", " groups = model.acd_param_groups(lm_over.get(\"lr\", 5e-4))\n", " else: # routed AlephLM: stage_books ride the geometric slow lane\n", " lr = lm_over.get(\"lr\", 5e-4)\n", " geom = [p for n, p in model.named_parameters()\n", " if \"stage_books\" in n or \"codebook\" in n or \"res_dec\" in n]\n", " gi = {id(p) for p in geom}\n", " rest = [p for p in model.parameters() if id(p) not in gi]\n", " groups = [dict(params=rest, lr=lr),\n", " dict(params=geom, lr=lr * 0.1, weight_decay=0.0)]\n", " opt = torch.optim.Adam(groups)\n", " t0, killed = time.time(), None\n", " model.train()\n", " for step in range(1, steps + 1):\n", " ids, tg = stream.sample(cfg.batch_size, cfg.seq_len, device)\n", " loss, _aux = model.forward_loss(ids, tg)\n", " opt.zero_grad()\n", " loss.backward()\n", " gn = torch.nn.utils.clip_grad_norm_(model.parameters(),\n", " max(loss.item(), 1.0))\n", " opt.step()\n", " if step % eval_every == 0 and not math.isfinite(loss.item()):\n", " killed = \"NaN/inf loss\"\n", " break\n", " # eval bpb on held batches\n", " model.eval()\n", " with torch.no_grad():\n", " tot, n = 0.0, 0\n", " for _ in range(4):\n", " ids, tg = stream.sample(cfg.batch_size, cfg.seq_len, device)\n", " l, _ = model.forward_loss(ids, tg)\n", " tot += l.item(); n += 1\n", " bpb = (tot / n) / math.log(2) / 3.0 # nats/trigram -> bits/byte\n", " mb = lm_marginal_bits(model, stream, cfg.seq_len, device,\n", " probe_steps=probe_steps, seed=spec.seed)\n", " H0 = mb[\"H0\"]\n", " cg = composition_gauges(model.conditioner.acd,\n", " torch.zeros(1, cfg.dim, device=device)) \\\n", " if False else _addr_gauges(mb[\"addresses\"])\n", " if hasattr(model, \"conditioner\"):\n", " st = model.conditioner.acd.codebook_stats()\n", " alpha_val = round(model.conditioner.alpha.item(), 4)\n", " else:\n", " st = _ns(\"routing_codebook_stats\", \"acd_attention\")(model)\n", " alpha_val = 0.0 # no whisper in routed arms\n", " return dict(arm_id=spec.arm_id(), op=spec.op, m=spec.m, K=spec.K,\n", " d_addr=spec.d_addr, freeze=spec.freeze, seed=spec.seed,\n", " tree_hard=spec.tree_hard, steps=steps,\n", " params=sum(p.numel() for p in model.parameters()),\n", " acc=round(mb[\"probe_acc\"], 4), ce_bits=round(bpb, 4),\n", " delivered_bits=round(H0 - bpb, 4), # bits above unigram\n", " cum_bits=round(mb[\"cumulative_bits\"][-1], 4),\n", " marginal_bits=_json.dumps(\n", " [round(v, 4) for v in mb[\"marginal_bits\"]]),\n", " redundancy=cg[\"redundancy\"],\n", " cancellation=cg[\"cancellation\"], stage_snr=0.0,\n", " dev_mean=round(sum(s[\"deviation\"] for s in st) / len(st), 4),\n", " rank_mean=round(sum(s[\"eff_rank\"] for s in st) / len(st), 3),\n", " alpha=alpha_val,\n", " killed=killed or \"\", wall_s=round(time.time() - t0, 1))\n", "\n", "\n", "@torch.no_grad()\n", "def _addr_gauges(addrs: Tensor) -> Dict[str, float]:\n", " \"\"\"redundancy/cancellation on cached (N, m, 2K) token addresses —\n", " same definitions as acd_probe.composition_gauges.\"\"\"\n", " N, m, W = addrs.shape\n", " K = W // 2\n", " if m == 1:\n", " return {\"redundancy\": 0.0, \"cancellation\": 0.0}\n", " flat = F.normalize(addrs, dim=-1)\n", " cos = torch.einsum(\"bmw,bnw->bmn\", flat, flat)\n", " iu = torch.triu_indices(m, m, offset=1)\n", " red = cos[:, iu[0], iu[1]].mean().item()\n", " net = addrs[..., :K] - addrs[..., K:]\n", " agree = (net.sign().sum(dim=1).abs() == m).float()\n", " canc = (((1 - agree) * net.abs().sum(1)).sum(-1)\n", " / net.abs().sum(dim=(1, 2)).clamp_min(1e-9)).mean().item()\n", " return {\"redundancy\": round(red, 4), \"cancellation\": round(canc, 4)}\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Smoke — synthetic stream, tiny LM, prod_m4 conditioner\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def _smoke():\n", " try:\n", " ArmSpec = _ns(\"ArmSpec\", \"acd_forge\")\n", " except Exception:\n", " # forge not pasted yet (it comes AFTER the adapter): a minimal shim\n", " # covering exactly the surface run_arm_L touches. Real runs always\n", " # use the forge grammar; the shim exists only for this smoke.\n", " from dataclasses import dataclass as _dc\n", " @_dc\n", " class ArmSpec:\n", " op: str\n", " m: int\n", " K: int = 0\n", " d_addr: int = 4\n", " freeze: str = \"free\"\n", " seed: int = 1234\n", " tree_hard: bool = False\n", " def resolve(self):\n", " if self.K == 0:\n", " self.K = max(2, 256 // (max(self.m, 1) * self.d_addr))\n", " if self.op == \"single\":\n", " self.m = 1\n", " return self\n", " def arm_id(self):\n", " return f\"smoke_{self.op}_m{self.m}_K{self.K}_s{self.seed}\"\n", " def to_acd(self, d_in, feature_dim=64):\n", " return ACDConfig(op=self.op, d_in=d_in, m=self.m, K=self.K,\n", " d_addr=self.d_addr, freeze=self.freeze,\n", " tree_hard=self.tree_hard, seed=self.seed,\n", " feature_dim=feature_dim)\n", " print(\" - ArmSpec shim in use (real grammar arrives with acd_forge)\")\n", " dev = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " stream = MarkovByteStream(seed=3)\n", " ids, tg = stream.sample(4, 32, dev)\n", " assert ids.shape == (4, 32, 3) and tg.shape == (4, 32, 3)\n", " print(f\" ✓ MarkovByteStream: trigram framing {tuple(ids.shape)}\")\n", "\n", " try:\n", " _ns(\"AlephLM\", \"aleph_lm\")\n", " except Exception:\n", " print(\" - aleph-lm cells not in namespace yet: arm smoke DEFERRED \"\n", " \"(paste cells 1-4, then re-run _smoke() — or it validates on \"\n", " \"the first phase3 arm)\")\n", " print(\"acd_lm_adapter smoke: PARTIAL (stream green, arms deferred)\")\n", " return\n", " over = smoke_overrides()\n", " spec = ArmSpec(op=\"prod\", m=4, K=8, seed=1234).resolve()\n", " row = run_arm_L(spec, steps=40, stream=stream, lm_over=over,\n", " device=dev, probe_steps=200, eval_every=20)\n", " assert math.isfinite(row[\"ce_bits\"]) and row[\"ce_bits\"] < 8.0\n", " assert isinstance(row[\"delivered_bits\"], float)\n", " assert -1.0 < row[\"delivered_bits\"] < 8.0 # rebased scale sanity\n", " assert 0.0 <= row[\"acc\"] <= 1.0 # probe top-1\n", " assert isinstance(row[\"alpha\"], float) # injection gate logged\n", " import json as _json\n", " marg = _json.loads(row[\"marginal_bits\"])\n", " assert len(marg) == 4 and all(v >= 0 for v in marg)\n", " print(f\" ✓ run_arm_L: bpb {row['ce_bits']:.3f} delivered \"\n", " f\"{row['delivered_bits']:.3f} held {row['cum_bits']:.3f} \"\n", " f\"{marg} red {row['redundancy']:.2f} \"\n", " f\"dev {row['dev_mean']:+.3f} rank {row['rank_mean']:.2f}\")\n", "\n", " # single-arm control path (m=1 gauges degenerate cleanly)\n", " s1 = ArmSpec(op=\"single\", m=1, K=32, seed=1234).resolve()\n", " r1 = run_arm_L(s1, steps=20, stream=stream, lm_over=over,\n", " device=dev, probe_steps=150, eval_every=20)\n", " assert r1[\"redundancy\"] == 0.0\n", " print(f\" ✓ single control: bpb {r1['ce_bits']:.3f}\")\n", "\n", " # alpha gate is learnable and moved\n", " print(\"acd_lm_adapter smoke: ALL GREEN\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "3X88nkU59dCS", "outputId": "6d588dd9-6cda-441d-c120-ed9d6f68566a" }, "execution_count": 12, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ MarkovByteStream: trigram framing (4, 32, 3)\n", " ✓ run_arm_L: bpb 5.598 delivered 0.039 held 0.036 [0.0193, 0.0, 0.0004, 0.0161] red 0.16 dev -0.010 rank 3.57\n", " ✓ single control: bpb 5.636\n", "acd_lm_adapter smoke: ALL GREEN\n" ] } ] }, { "cell_type": "code", "source": [ "# ============================================================\n", "# acd_forge.py — exp_011 automation\n", "# Grammar -> generator (auto budget-twins + SUM controls) -> rung\n", "# scheduler (successive halving) -> kill rules -> ledger -> HF push.\n", "#\n", "# The Captain reviews VERDICTS, not arms: every promote/park/kill is\n", "# logged with the gauge values that caused it.\n", "#\n", "# Rungs v1: P-200 -> P-1000 (Tier-P implemented). Tier-L rungs raise\n", "# NotImplementedError at a clean seam until acd_lm_adapter.py lands.\n", "# Lane parallelism (vmap seed-groups): DEFERRED v1.1 — sequential is\n", "# correct and rung0 runs single-seed; the bit-equivalence gate applies\n", "# when lanes ship, not before.\n", "#\n", "# Repo: AbstractPhil/geolip-aleph-differentiation (exp011/ prefix)\n", "# Paste order: acd_structures.py -> acd_probe.py -> acd_forge.py\n", "# ============================================================\n", "from __future__ import annotations\n", "import csv, hashlib, json, math, os, time\n", "from dataclasses import asdict, dataclass, field\n", "from typing import Dict, List, Optional, Tuple\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", "try:\n", " from acd_structures import ACDConfig, ACDStructure, match_budget, OPS\n", " from acd_probe import (BubbleConfig, NestedBubbles, marginal_bits,\n", " composition_gauges)\n", "except ImportError:\n", " pass # notebook paste mode\n", "\n", "def _ns(name: str, module: str):\n", " \"\"\"Cross-cell resolver. Pasted Colab cells share ONE namespace and are\n", " not importable modules — so resolve names from globals() first (paste\n", " mode), then fall back to a real import (script/module mode).\"\"\"\n", " if name in globals():\n", " return globals()[name]\n", " import importlib\n", " return getattr(importlib.import_module(module), name)\n", "\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# BaseConfig\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "REPO_ID = \"AbstractPhil/geolip-aleph-differentiation\"\n", "EXP_PREFIX = \"exp011\"\n", "BUDGET_KD = 64 * 4 # single-aleph reference: K=64, D=4 codebook floats\n", "\n", "@dataclass\n", "class ForgeConfig:\n", " out_dir: str = \"exp011\"\n", " rungs: Tuple[Tuple[str, int, float], ...] = (\n", " (\"P\", 200, 1 / 3), (\"P\", 1000, 1 / 3),\n", " (\"L\", 1000, 1 / 3), (\"L\", 5000, 1.0))\n", " lr: float = 3e-3\n", " batch: int = 512\n", " eval_every: int = 50\n", " probe_steps: int = 250 # marginal-bits probe budget per prefix\n", " n_train: int = 8192 # fixed shared dataset per screen\n", " n_eval: int = 4096\n", " bubble: \"BubbleConfig\" = None # set in __post_init__ (shared task!)\n", " push: bool = True # False = dry run (no network)\n", " lm: Optional[Dict] = None # Tier-L AlephLM overrides (None -> tier_l recipe)\n", " lm_corpus: str = \"wikitext-103-raw-v1\" # 'synthetic' -> Markov smoke stream\n", " device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " seed: int = 1234\n", "\n", " exp_prefix: str = \"\" # HF push prefix; defaults to out_dir\n", "\n", " def __post_init__(self):\n", " if not self.exp_prefix:\n", " self.exp_prefix = self.out_dir\n", " if self.bubble is None:\n", " self.bubble = BubbleConfig(d_data=32, branching=(4, 4, 4),\n", " sep0=6.0, sep_decay=0.45,\n", " noise=0.35, seed=97)\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Grammar — an arm is a JSON spec; its hash is its identity\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "@dataclass\n", "class ArmSpec:\n", " op: str\n", " m: int\n", " d_addr: int = 4\n", " freeze: str = \"free\"\n", " seed: int = 1234\n", " tree_hard: bool = False\n", " budget_kd: int = BUDGET_KD\n", " K: int = 0 # 0 -> solved by match_budget\n", " coupling: str = \"whisper\" # Tier-L head coupling (3b ladder)\n", "\n", " def resolve(self) -> \"ArmSpec\":\n", " if self.K == 0:\n", " self.K = match_budget(self.op, self.m, self.d_addr,\n", " self.budget_kd)\n", " if self.op == \"single\":\n", " self.m = 1\n", " return self\n", "\n", " def arm_id(self) -> str:\n", " d = asdict(self)\n", " if d.get(\"coupling\", \"whisper\") == \"whisper\":\n", " d.pop(\"coupling\", None) # LEGACY-STABLE: old ids reproduce\n", " s = json.dumps(d, sort_keys=True)\n", " cpl = \"\" if self.coupling == \"whisper\" else f\"_{self.coupling}\"\n", " return f\"{self.op}_m{self.m}_K{self.K}\" \\\n", " f\"{'_hard' if self.tree_hard else ''}{cpl}\" \\\n", " f\"_{self.freeze}_s{self.seed}_{hashlib.sha1(s.encode()).hexdigest()[:6]}\"\n", "\n", " def to_acd(self, d_in: int, feature_dim: int = 64) -> \"ACDConfig\":\n", " return ACDConfig(op=self.op, d_in=d_in, m=self.m, K=self.K,\n", " d_addr=self.d_addr, freeze=self.freeze,\n", " tree_hard=self.tree_hard, seed=self.seed,\n", " feature_dim=feature_dim)\n", "\n", "\n", "def generate_arms(ops: List[str], ms: List[int], freezes: List[str],\n", " seeds: List[int], existing_ids: Optional[set] = None,\n", " tree_both_modes: bool = True) -> List[ArmSpec]:\n", " \"\"\"Grid expansion + the two mandatory scientific controls:\n", " (1) budget-matched SINGLE twin per (freeze, seed),\n", " (2) SUM control at every m present (the divergence reference).\n", " Dedup against existing ledger ids.\"\"\"\n", " arms: List[ArmSpec] = []\n", " for op in ops:\n", " for m in ms:\n", " if op == \"single\":\n", " continue\n", " for fz in freezes:\n", " for sd in seeds:\n", " if op == \"tree\" and tree_both_modes:\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd,\n", " tree_hard=False))\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd,\n", " tree_hard=True))\n", " else:\n", " arms.append(ArmSpec(op, m, freeze=fz, seed=sd))\n", " for fz in freezes: # control (1)\n", " for sd in seeds:\n", " arms.append(ArmSpec(\"single\", 1, freeze=fz, seed=sd))\n", " if \"sum\" not in ops: # control (2)\n", " for m in ms:\n", " for fz in freezes:\n", " for sd in seeds:\n", " arms.append(ArmSpec(\"sum\", m, freeze=fz, seed=sd))\n", " out, seen = [], set(existing_ids or ())\n", " for a in arms:\n", " a.resolve()\n", " if a.arm_id() not in seen:\n", " seen.add(a.arm_id())\n", " out.append(a)\n", " return out\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Kill rules — fire inside a rung; every kill carries its reason\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def check_kill(loss_val: float, grad_norm: float,\n", " stats: List[Dict]) -> Optional[str]:\n", " if not math.isfinite(loss_val):\n", " return \"NaN/inf loss\"\n", " if grad_norm > 1e3:\n", " return f\"grad blowup |g|={grad_norm:.1f}\"\n", " for i, s in enumerate(stats):\n", " ceiling = min(4.0, s.get(\"eff_rank_ceiling\", 4.0))\n", " if s[\"eff_rank\"] < 0.5 * ceiling:\n", " return f\"rank collapse stage{i} rank={s['eff_rank']:.2f}\"\n", " return None\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Tier-P trainer — one arm, one rung\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_arm_P(spec: ArmSpec, steps: int, fc: ForgeConfig,\n", " data: Tuple) -> Dict[str, object]:\n", " xtr, ytr, ltr, xev, yev, lev = data\n", " dev = fc.device\n", " torch.manual_seed(spec.seed)\n", " net = ACDStructure(spec.to_acd(d_in=fc.bubble.d_data)).to(dev)\n", " head = nn.Linear(net.cfg.feature_dim, fc.bubble.n_leaves).to(dev)\n", " opt = torch.optim.Adam(list(net.parameters()) + list(head.parameters()),\n", " lr=fc.lr) # pure Adam, statute\n", " t0, killed = time.time(), None\n", " n = xtr.shape[0]\n", " for step in range(1, steps + 1):\n", " idx = torch.randint(0, n, (fc.batch,), device=dev)\n", " feats, _ = net(xtr[idx])\n", " loss = F.cross_entropy(head(feats), ytr[idx])\n", " opt.zero_grad()\n", " loss.backward()\n", " gn = torch.nn.utils.clip_grad_norm_(\n", " list(net.parameters()) + list(head.parameters()),\n", " max(loss.item(), 1.0)) # standing clip rule\n", " opt.step()\n", " if step % fc.eval_every == 0 or step == steps:\n", " reason = check_kill(loss.item(), gn.item(), net.codebook_stats())\n", " if reason:\n", " killed = reason\n", " break\n", " # rung-end gauges (also computed for killed arms — the corpse is data)\n", " net.eval()\n", " with torch.no_grad():\n", " fe, _ = net(xev)\n", " ce_bits = F.cross_entropy(head(fe), yev).item() / math.log(2)\n", " acc = (head(fe).argmax(-1) == yev).float().mean().item()\n", " # probe budget scales with class count: 250 steps underfits a\n", " # 256-way linear readout (measured: single held 3.11 vs delivered\n", " # 5.34 at 2b). Within-op comparisons survive the bias; the\n", " # held-vs-delivered gap does not, so we feed the probe properly.\n", " psteps = max(fc.probe_steps, int(2.5 * fc.bubble.n_leaves))\n", " mb = marginal_bits(net, xev, yev, fc.bubble.n_leaves,\n", " probe_steps=psteps, seed=spec.seed)\n", " cg = composition_gauges(net, xev[:1024], cluster=lev[:1024, 0])\n", " st = net.codebook_stats()\n", " row = dict(arm_id=spec.arm_id(), op=spec.op, m=spec.m, K=spec.K,\n", " d_addr=spec.d_addr, freeze=spec.freeze, seed=spec.seed,\n", " tree_hard=spec.tree_hard, steps=steps,\n", " params=net.param_count(),\n", " acc=round(acc, 4), ce_bits=round(ce_bits, 4),\n", " cum_bits=round(mb[\"cumulative_bits\"][-1], 4),\n", " delivered_bits=round(\n", " math.log2(fc.bubble.n_leaves) - ce_bits, 4),\n", " marginal_bits=json.dumps(\n", " [round(v, 4) for v in mb[\"marginal_bits\"]]),\n", " redundancy=round(cg[\"redundancy\"], 4),\n", " cancellation=round(cg[\"cancellation\"], 4),\n", " stage_snr=round(cg.get(\"stage_snr\", 0.0), 4),\n", " dev_mean=round(sum(s[\"deviation\"] for s in st) / len(st), 4),\n", " rank_mean=round(sum(s[\"eff_rank\"] for s in st) / len(st), 3),\n", " killed=killed or \"\", wall_s=round(time.time() - t0, 1))\n", " return row\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Ledger + HF push\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "_CSV_COLS = [\"rung\", \"arm_id\", \"op\", \"m\", \"K\", \"d_addr\", \"freeze\", \"seed\",\n", " \"tree_hard\", \"steps\", \"params\", \"acc\", \"ce_bits\", \"cum_bits\",\n", " \"delivered_bits\",\n", " \"marginal_bits\", \"redundancy\", \"cancellation\", \"stage_snr\",\n", " \"dev_mean\", \"rank_mean\", \"alpha\",\n", " \"verdict\", \"reason\", \"killed\", \"wall_s\"]\n", "\n", "def ledger_append(fc: ForgeConfig, rows: List[Dict]):\n", " os.makedirs(fc.out_dir, exist_ok=True)\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " new = not os.path.exists(path)\n", " if new:\n", " cols = _CSV_COLS\n", " else: # header-aware: never shift columns of a pre-existing ledger\n", " with open(path) as f:\n", " cols = next(csv.reader(f))\n", " with open(path, \"a\", newline=\"\") as f:\n", " w = csv.DictWriter(f, fieldnames=cols, extrasaction=\"ignore\",\n", " restval=\"\")\n", " if new:\n", " w.writeheader()\n", " w.writerows(rows)\n", "\n", "\n", "def ledger_rows(fc: ForgeConfig) -> List[Dict]:\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " if not os.path.exists(path):\n", " return []\n", " with open(path) as f:\n", " return list(csv.DictReader(f))\n", "\n", "\n", "def ledger_ids(fc: ForgeConfig) -> set:\n", " path = os.path.join(fc.out_dir, \"results.csv\")\n", " if not os.path.exists(path):\n", " return set()\n", " with open(path) as f:\n", " return {r[\"arm_id\"] for r in csv.DictReader(f)}\n", "\n", "\n", "def write_sweep_md(fc: ForgeConfig, all_rows: List[Dict]):\n", " rows = sorted(all_rows, key=lambda r: -float(r[\"cum_bits\"]))\n", " lines = [\"# exp_011 ACD — sweep leaderboard\", \"\",\n", " f\"Task: nested bubbles {fc.bubble.branching} \"\n", " f\"({fc.bubble.n_leaves} leaves, \"\n", " f\"{math.log2(fc.bubble.n_leaves):.1f} bits available)\", \"\",\n", " \"| arm | rung | cum bits | marginal | acc | red | canc | dev | rank | verdict |\",\n", " \"|---|---|---|---|---|---|---|---|---|---|\"]\n", " for r in rows:\n", " lines.append(\n", " f\"| `{r['arm_id']}` | {r['rung']} | **{r['cum_bits']}** \"\n", " f\"| {r['marginal_bits']} | {r['acc']} | {r['redundancy']} \"\n", " f\"| {r['cancellation']} | {r['dev_mean']} | {r['rank_mean']} \"\n", " f\"| {r['verdict']}{(' — ' + r['reason']) if r['reason'] else ''} |\")\n", " with open(os.path.join(fc.out_dir, \"SWEEP.md\"), \"w\") as f:\n", " f.write(\"\\n\".join(lines) + \"\\n\")\n", "\n", "\n", "def hf_push(fc: ForgeConfig):\n", " if not fc.push:\n", " print(\"[push] dry run — skipped\")\n", " return\n", " from huggingface_hub import HfApi, create_repo\n", " create_repo(REPO_ID, repo_type=\"model\", exist_ok=True)\n", " api = HfApi()\n", " for name in (\"results.csv\", \"SWEEP.md\", \"verdicts.jsonl\"):\n", " p = os.path.join(fc.out_dir, name)\n", " if os.path.exists(p):\n", " api.upload_file(path_or_fileobj=p,\n", " path_in_repo=f\"{fc.exp_prefix}/{name}\",\n", " repo_id=REPO_ID, repo_type=\"model\",\n", " commit_message=f\"{fc.exp_prefix}: {name}\")\n", " print(f\"[push] {fc.exp_prefix}/ -> {REPO_ID} ✓\")\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Scheduler — successive halving with logged verdicts\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def run_screen(arms: List[ArmSpec], fc: ForgeConfig,\n", " rungs: Optional[List[int]] = None,\n", " protect_ops: Tuple[str, ...] = ()) -> List[Dict]:\n", " \"\"\"Run arms through the configured rungs; keep top keep_frac by\n", " cum_bits per rung (kills never advance). Returns all ledger rows.\"\"\"\n", " g = torch.Generator().manual_seed(fc.seed)\n", " bub = NestedBubbles(fc.bubble)\n", " xtr, ytr, ltr = bub.sample(fc.n_train, device=fc.device)\n", " xev, yev, lev = bub.sample(fc.n_eval, device=fc.device)\n", " data = (xtr, ytr, ltr, xev, yev, lev)\n", "\n", " alive = list(arms)\n", " if not alive:\n", " prior = ledger_rows(fc)\n", " print(f\"[screen] nothing queued — ledger already holds \"\n", " f\"{len(prior)} rows in {fc.out_dir}/results.csv; \"\n", " f\"call report(fc) to view standings.\")\n", " return prior\n", " all_rows: List[Dict] = []\n", " vpath = os.path.join(fc.out_dir, \"verdicts.jsonl\")\n", " os.makedirs(fc.out_dir, exist_ok=True)\n", " for ri, (tier, steps, keep) in enumerate(fc.rungs):\n", " if rungs is not None and ri not in rungs:\n", " continue\n", " if tier == \"L\":\n", " try:\n", " run_arm_L = _ns(\"run_arm_L\", \"acd_lm_adapter\")\n", " tier_l_overrides = _ns(\"tier_l_overrides\",\n", " \"acd_lm_adapter\")\n", " MarkovByteStream = _ns(\"MarkovByteStream\",\n", " \"acd_lm_adapter\")\n", " except Exception as e:\n", " raise RuntimeError(\n", " \"Tier-L needs acd_lm_adapter in the namespace — \"\n", " \"paste it (after the aleph-lm cells) first.\") from e\n", " if not hasattr(fc, \"_lm_stream\"):\n", " if fc.lm_corpus == \"synthetic\":\n", " fc._lm_stream = MarkovByteStream(seed=fc.seed)\n", " else:\n", " TrigramStream = _ns(\"TrigramStream\", \"aleph_lm\")\n", " fc._lm_stream = TrigramStream(\n", " fc.lm_corpus, max_corpus_bytes=100_000_000,\n", " seed=fc.seed)\n", " lm_over = fc.lm or tier_l_overrides()\n", " run_L = lambda sp, st: run_arm_L(\n", " sp, st, fc._lm_stream, lm_over, fc.device,\n", " probe_steps=max(fc.probe_steps, 640),\n", " eval_every=fc.eval_every)\n", " cached = {r[\"arm_id\"]: r for r in ledger_rows(fc)\n", " if r.get(\"rung\") == str(ri)}\n", " n_hit = sum(1 for s in alive if s.arm_id() in cached)\n", " print(f\"\\n[rung {ri}] tier={tier} steps={steps} \"\n", " f\"arms={len(alive)} (cached {n_hit}, \"\n", " f\"fresh {len(alive) - n_hit})\")\n", " rows, fresh = [], []\n", " for k, spec in enumerate(alive):\n", " aid = spec.arm_id()\n", " if aid in cached: # RESUME: reuse the ledger row\n", " rows.append(cached[aid])\n", " print(f\" [{k+1}/{len(alive)}] {aid:34s} cached \"\n", " f\"(bits {cached[aid]['cum_bits']})\")\n", " continue\n", " row = (run_arm_P(spec, steps, fc, data) if tier == \"P\"\n", " else run_L(spec, steps))\n", " row[\"rung\"] = ri\n", " rows.append(row)\n", " fresh.append(row)\n", " print(f\" [{k+1}/{len(alive)}] {row['arm_id']:34s} \"\n", " f\"held {row['cum_bits']:.2f} ce {row['ce_bits']:.3f} \"\n", " f\"dlv {row['delivered_bits']:.2f} \"\n", " f\"{row['marginal_bits']} \"\n", " f\"acc {row['acc']:.3f} red {row['redundancy']:.2f}\"\n", " f\"{' a ' + format(row['alpha'], '.3f') if 'alpha' in row else ''}\"\n", " f\"{' KILLED: ' + row['killed'] if row['killed'] else ''}\")\n", " # verdicts\n", " survivors = [r for r in rows if not r[\"killed\"]]\n", " survivors.sort(key=lambda r: -float(r[\"cum_bits\"]))\n", " n_keep = max(1, int(len(survivors) * keep))\n", " promoted = {r[\"arm_id\"] for r in survivors[:n_keep]}\n", " # science over leaderboard: controls ride every rung so the\n", " # divergence gate is actually tested at full training depth\n", " shielded = {r[\"arm_id\"] for r in survivors\n", " if r[\"op\"] in protect_ops}\n", " promoted |= shielded\n", " with open(vpath, \"a\") as vf:\n", " for r in rows:\n", " if r[\"killed\"]:\n", " r[\"verdict\"], r[\"reason\"] = \"KILL\", r[\"killed\"]\n", " elif r[\"arm_id\"] in promoted:\n", " why = (\"protected control\"\n", " if r[\"op\"] in protect_ops\n", " and r[\"arm_id\"] not in\n", " {s[\"arm_id\"] for s in survivors[:n_keep]}\n", " else f\"top {n_keep}/{len(survivors)} by cum_bits\")\n", " r[\"verdict\"], r[\"reason\"] = \"PROMOTE\", why\n", " else:\n", " r[\"verdict\"], r[\"reason\"] = \"PARK\", \"below keep line\"\n", " if any(r is fr for fr in fresh):\n", " vf.write(json.dumps(r) + \"\\n\")\n", " ledger_append(fc, fresh)\n", " all_rows += rows\n", " write_sweep_md(fc, all_rows)\n", " hf_push(fc)\n", " by_id = {a.arm_id(): a for a in alive}\n", " alive = [by_id[i] for i in promoted if i in by_id]\n", " if not alive:\n", " print(\"[screen] no survivors — stopping\")\n", " break\n", " return all_rows\n", "\n", "\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "# Activation — dual-mode (script main / notebook cell)\n", "# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", "\n", "def phase2_screen(push: bool = True):\n", " \"\"\"The Phase-2 mass screen: full operator grid, rung0+rung1.\"\"\"\n", " fc = ForgeConfig(push=push)\n", " ops = [\"sum\", \"gate\", \"res\", \"prod\", \"tree\", \"cross\", \"anneal\"]\n", " arms = generate_arms(ops, ms=[2, 3, 4], freezes=[\"free\", \"spread\"],\n", " seeds=[1234]) # resume-safe: cache skips reruns\n", " print(f\"[forge] {len(arms)} arms queued (incl. controls)\")\n", " return run_screen(arms, fc, rungs=[0, 1])\n", "\n", "\n", "def phase2b_screen(push: bool = True):\n", " \"\"\"Phase 2b: divergence hunt with headroom. 256-leaf task (8 bits),\n", " m up to 8, two seeds, SUM + SINGLE protected to full training depth.\n", " Own ledger (exp011b/) — the task changed, so rows must not mix.\"\"\"\n", " fc = ForgeConfig(out_dir=\"exp011b\", push=push,\n", " n_train=16384, n_eval=8192, batch=1024,\n", " bubble=BubbleConfig(d_data=32, branching=(4, 4, 4, 4),\n", " sep0=6.0, sep_decay=0.5,\n", " noise=0.3, seed=97))\n", " ops = [\"sum\", \"gate\", \"res\", \"prod\", \"cross\"] # anneal parked (rung-0\n", " # verdict: clone stages); tree awaits the budget-accounting decision\n", " arms = generate_arms(ops, ms=[2, 4, 6, 8], freezes=[\"free\", \"spread\"],\n", " seeds=[1234, 5678])\n", " print(f\"[forge] {len(arms)} arms queued (sum+single protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", "\n", "\n", "def tree_dual_arms(ms, freezes, seeds, budget_kd: int = BUDGET_KD,\n", " K0: int = 8, d_addr: int = 4):\n", " \"\"\"The 'both' ruling: every tree config in BOTH accountings.\n", " total — storage-matched: K = match_budget('tree', ...) (soft+hard)\n", " active — MoE convention (HARD only; soft touches all branches):\n", " per-input params = router K0*D + ONE branch K*D\n", " => K_active = (budget - K0*D) // d_addr.\"\"\"\n", " arms = []\n", " K_act = max(2, (budget_kd - K0 * d_addr) // d_addr)\n", " for m in ms:\n", " for fz in freezes:\n", " for sd in seeds:\n", " for hard in (False, True): # total accounting\n", " arms.append(ArmSpec(\"tree\", m, d_addr=d_addr, freeze=fz,\n", " seed=sd, tree_hard=hard).resolve())\n", " arms.append(ArmSpec(\"tree\", m, d_addr=d_addr, freeze=fz,\n", " seed=sd, tree_hard=True,\n", " K=K_act)) # active accounting (hard)\n", " return arms\n", "\n", "\n", "def phase4_screen(push: bool = True):\n", " \"\"\"Phase 4 — composition WHERE INFORMATION IS CREATED: composed\n", " micro-addresses as the attention feature map (exp011R). single = the\n", " stock router (parity-gated control); sum = unconditioned width-fair\n", " control; prod/res = the conditioned operators. K field = K_total=64\n", " (feature width AND codebook budget, matched by construction).\"\"\"\n", " tier_l_overrides = _ns(\"tier_l_overrides\", \"acd_lm_adapter\")\n", " fc = ForgeConfig(out_dir=\"exp011R\", push=push,\n", " rungs=((\"L\", 1000, 0.6), (\"L\", 5000, 1.0)),\n", " lm=tier_l_overrides())\n", " arms = []\n", " for sd in (1234, 5678):\n", " arms.append(ArmSpec(\"single\", 1, K=64, freeze=\"free\", seed=sd,\n", " coupling=\"routed\"))\n", " for op in (\"sum\", \"prod\", \"res\"):\n", " for m in (2, 4):\n", " arms.append(ArmSpec(op, m, K=64, freeze=\"free\", seed=sd,\n", " coupling=\"routed\"))\n", " arms = [a.resolve() for a in arms]\n", " print(f\"[forge] Phase 4: {len(arms)} routed arms (single+sum protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"single\", \"sum\"))\n", "\n", "\n", "def phase3b_screen(push: bool = True):\n", " \"\"\"Phase 3b — head-through-structure: the coupling ladder on the\n", " language champion (res_m8), sum control + singles protected.\n", " apmix_src arms run the hybrid recipe (corpus bank); the rest byte-head.\n", " Own ledger exp011H.\"\"\"\n", " tier_l_overrides = _ns(\"tier_l_overrides\", \"acd_lm_adapter\")\n", " hybrid = _ns(\"tier_l_hybrid_overrides\", \"acd_lm_adapter\")\n", " fc = ForgeConfig(out_dir=\"exp011H\", push=push,\n", " rungs=((\"L\", 1000, 0.6), (\"L\", 5000, 1.0)),\n", " lm=tier_l_overrides())\n", " fc_h = ForgeConfig(out_dir=\"exp011H\", push=push,\n", " rungs=fc.rungs, lm=hybrid())\n", " arms, arms_h = [], []\n", " for sd in (1234, 5678):\n", " for cpl in (\"whisper\", \"concat\", \"gate\"):\n", " arms.append(ArmSpec(\"res\", 8, freeze=\"free\", seed=sd,\n", " coupling=cpl).resolve())\n", " arms.append(ArmSpec(\"sum\", 8, freeze=\"free\", seed=sd).resolve())\n", " arms.append(ArmSpec(\"single\", 1, freeze=\"free\", seed=sd).resolve())\n", " arms_h.append(ArmSpec(\"res\", 8, freeze=\"free\", seed=sd,\n", " coupling=\"apmix_src\").resolve())\n", " print(f\"[forge] 3b: {len(arms)} byte-head + {len(arms_h)} hybrid arms\")\n", " rows = run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", " rows += run_screen(arms_h, fc_h, rungs=[0, 1])\n", " return rows\n", "\n", "\n", "def phase3_screen(push: bool = True):\n", " \"\"\"Phase 3: Tier-P finalists onto the 6.75M LM (WikiText-103).\n", " prod/res at their knees + sum control + singles, byte head first\n", " (bank-free); hybrid/apmix arms follow once the byte read is banked.\n", " ce_bits column = bits/byte; delivered_bits = 8 - bpb.\"\"\"\n", " tier_l_overrides = _ns(\"tier_l_overrides\", \"acd_lm_adapter\")\n", " fc = ForgeConfig(out_dir=\"exp011L\", push=push,\n", " rungs=((\"L\", 1000, 0.5), (\"L\", 5000, 1.0)),\n", " lm=tier_l_overrides())\n", " arms = generate_arms([\"prod\", \"res\", \"sum\"], ms=[8],\n", " freezes=[\"free\"], seeds=[1234, 5678])\n", " arms += generate_arms([\"res\"], ms=[16], freezes=[\"free\"],\n", " seeds=[1234, 5678],\n", " existing_ids={a.arm_id() for a in arms})\n", " print(f\"[forge] {len(arms)} Tier-L arms queued (sum+single protected)\")\n", " return run_screen(arms, fc, rungs=[0, 1],\n", " protect_ops=(\"sum\", \"single\"))\n", "\n", "\n", "def report(fc: Optional[ForgeConfig] = None, top: int = 15):\n", " \"\"\"Print standings from the ledger without running anything.\"\"\"\n", " fc = fc or ForgeConfig(push=False)\n", " rows = ledger_rows(fc)\n", " if not rows:\n", " print(f\"[report] no ledger at {fc.out_dir}/results.csv\")\n", " return rows\n", " rows.sort(key=lambda r: (-int(r[\"rung\"]), -float(r[\"cum_bits\"])))\n", " print(f\"[report] {len(rows)} rows — top {top}:\")\n", " for r in rows[:top]:\n", " k = \" KILLED:\" + r[\"killed\"] if r[\"killed\"] else \"\"\n", " print(f\" r{r['rung']} {r['arm_id']:36s} bits {r['cum_bits']:>7s} \"\n", " f\"acc {r['acc']:>6s} red {r['redundancy']:>6s} \"\n", " f\"{r['verdict']}{k}\")\n", " return rows\n", "\n", "\n", "def _smoke():\n", " # Self-cleaning: Colab working dirs PERSIST across sessions, and a\n", " # stale smoke ledger turns fresh-arm tests into cache hits. The\n", " # smoke always starts from a swept room; resume behavior is tested\n", " # WITHIN the smoke (first pass trains, second pass must cache).\n", " import shutil\n", " for _d in (\"exp011_smoke\", \"exp011_smoke_prot\"):\n", " shutil.rmtree(_d, ignore_errors=True)\n", " fc = ForgeConfig(push=False, n_train=1536, n_eval=768, batch=256,\n", " probe_steps=120, eval_every=25,\n", " rungs=((\"P\", 60, 0.5), (\"P\", 120, 1.0)),\n", " out_dir=\"exp011_smoke\")\n", " arms = generate_arms([\"sum\", \"res\", \"prod\"], ms=[3], freezes=[\"free\"],\n", " seeds=[1234])\n", " ids = [a.arm_id() for a in arms]\n", " assert len(ids) == len(set(ids)), \"id collision\"\n", " assert any(a.op == \"single\" for a in arms), \"budget twin missing\"\n", " print(f\" ✓ generator: {len(arms)} arms \"\n", " f\"({[a.op for a in arms]})\")\n", " # kill-rule unit checks\n", " assert check_kill(float(\"nan\"), 1.0, []) == \"NaN/inf loss\"\n", " assert check_kill(1.0, 5e3, []).startswith(\"grad blowup\")\n", " assert check_kill(1.0, 1.0, [{\"eff_rank\": 1.2}]).startswith(\"rank collapse\")\n", " assert check_kill(1.0, 1.0, [{\"eff_rank\": 3.8}]) is None\n", " print(\" ✓ kill rules fire correctly\")\n", " rows = run_screen(arms, fc, rungs=[0, 1])\n", " assert os.path.exists(os.path.join(fc.out_dir, \"results.csv\"))\n", " assert os.path.exists(os.path.join(fc.out_dir, \"SWEEP.md\"))\n", " assert os.path.exists(os.path.join(fc.out_dir, \"verdicts.jsonl\"))\n", " assert all(r[\"verdict\"] in (\"PROMOTE\", \"PARK\", \"KILL\") for r in rows)\n", " # RESUME test: re-run same arms -> all cached, nothing trains;\n", " # add one new op -> only it trains, union re-ranked\n", " n_ledger_before = len(ledger_rows(fc))\n", " t0 = time.time()\n", " rows2 = run_screen(arms, fc, rungs=[0, 1])\n", " assert time.time() - t0 < 20, \"resume re-trained instead of caching\"\n", " assert len(ledger_rows(fc)) == n_ledger_before, \"cache appended rows\"\n", " r0 = [r for r in rows2 if str(r[\"rung\"]) == \"0\"]\n", " assert r0 and all(isinstance(r[\"cum_bits\"], str) for r in r0), \\\n", " \"rung0 rows were re-trained (fresh rows are floats)\"\n", " arms3 = generate_arms([\"sum\", \"res\", \"prod\", \"gate\"], ms=[3],\n", " freezes=[\"free\"], seeds=[1234])\n", " rows3 = run_screen(arms3, fc, rungs=[0])\n", " fresh_gate = [r for r in rows3 if r[\"op\"] == \"gate\"]\n", " assert len(fresh_gate) == 1 and isinstance(fresh_gate[0][\"cum_bits\"], float)\n", " assert isinstance(fresh_gate[0].get(\"delivered_bits\"), float), \\\n", " \"delivered_bits missing from fresh rows\"\n", " print(\" ✓ resume: full-cache fast path + mixed cache/fresh both work\")\n", " # protection: sum rides to rung1 despite bottom rank\n", " fcp = ForgeConfig(push=False, n_train=1024, n_eval=512, batch=256,\n", " probe_steps=80, eval_every=25,\n", " rungs=((\"P\", 40, 0.34), (\"P\", 60, 1.0)),\n", " out_dir=\"exp011_smoke_prot\")\n", " armsp = generate_arms([\"sum\", \"res\", \"prod\"], ms=[3], freezes=[\"free\"],\n", " seeds=[1234])\n", " rowsp = run_screen(armsp, fcp, rungs=[0, 1], protect_ops=(\"sum\",))\n", " r1_ops = {r[\"op\"] for r in rowsp if str(r[\"rung\"]) == \"1\"}\n", " assert \"sum\" in r1_ops, f\"protected sum culled: rung1 ops {r1_ops}\"\n", " prot = [r for r in rowsp if r[\"op\"] == \"sum\" and str(r[\"rung\"]) == \"0\"]\n", " assert any(\"protected\" in r[\"reason\"] or \"top\" in r[\"reason\"]\n", " for r in prot)\n", " print(\" ✓ protected controls ride the keep-line\")\n", " # Tier-L path (synthetic stream; skipped if adapter not pasted yet)\n", " try:\n", " smoke_overrides = _ns(\"smoke_overrides\", \"acd_lm_adapter\")\n", " import shutil as _sh\n", " _sh.rmtree(\"exp011_smoke_L\", ignore_errors=True)\n", " fcl = ForgeConfig(out_dir=\"exp011_smoke_L\", push=False,\n", " rungs=((\"L\", 25, 1.0),), probe_steps=150,\n", " eval_every=20, lm=smoke_overrides(),\n", " lm_corpus=\"synthetic\")\n", " armsl = generate_arms([\"prod\"], ms=[2], freezes=[\"free\"],\n", " seeds=[1234])\n", " rowsl = run_screen(armsl, fcl, rungs=[0])\n", " fr = [r for r in rowsl if isinstance(r[\"ce_bits\"], float)]\n", " assert fr and all(r[\"ce_bits\"] < 8.0 for r in fr)\n", " assert all(isinstance(r[\"delivered_bits\"], float) for r in fr)\n", " print(\" ✓ Tier-L rung: bpb finite, delivered logged, \"\n", " \"ledger schema intact\")\n", " except Exception:\n", " print(\" - Tier-L smoke skipped (acd_lm_adapter not in namespace)\")\n", " # empty-queue guard returns ledger instead of []\n", " empty = run_screen([], fc, rungs=[0])\n", " assert len(empty) > 0, \"empty guard returned nothing\"\n", " print(\" ✓ empty-queue guard returns ledger standings\")\n", " print(\"acd_forge smoke: ALL GREEN — next: phase2_screen()\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " # Notebook cells execute as __main__, so the smoke fires on paste too —\n", " # deliberate: pasting a cell IS the verification step in the Colab flow\n", " # (shared namespace, paste order structures -> probe -> forge).\n", " # Heavy entry points (phase2_screen) are never wired here; call them.\n", " _smoke()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "EBZVB6LH9fm9", "outputId": "558d5dab-0d61-4446-8761-16195c06164d" }, "execution_count": 13, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " ✓ generator: 4 arms (['sum', 'res', 'prod', 'single'])\n", " ✓ kill rules fire correctly\n", "\n", "[rung 0] tier=P steps=60 arms=4 (cached 0, fresh 4)\n", " [1/4] sum_m3_K21_free_s1234_7faffe held 3.68 ce 4.499 dlv 1.50 [2.9094, 0.2359, 0.5361] acc 0.111 red 0.23\n", " [2/4] res_m3_K21_free_s1234_1925f3 held 4.15 ce 3.174 dlv 2.83 [3.3556, 0.216, 0.5828] acc 0.310 red 0.08\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 held 4.13 ce 3.063 dlv 2.94 [3.0398, 0.5619, 0.5312] acc 0.253 red 0.05\n", " [4/4] single_m1_K64_free_s1234_194318 held 2.95 ce 4.330 dlv 1.67 [2.9487] acc 0.185 red 0.00\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=120 arms=2 (cached 0, fresh 2)\n", " [1/2] prod_m3_K21_free_s1234_c5d5b7 held 4.36 ce 1.970 dlv 4.03 [3.2752, 0.5116, 0.5715] acc 0.453 red 0.05\n", " [2/2] res_m3_K21_free_s1234_1925f3 held 4.41 ce 1.938 dlv 4.06 [3.4941, 0.2551, 0.6582] acc 0.466 red 0.06\n", "[push] dry run — skipped\n", "\n", "[rung 0] tier=P steps=60 arms=4 (cached 4, fresh 0)\n", " [1/4] sum_m3_K21_free_s1234_7faffe cached (bits 3.6814)\n", " [2/4] res_m3_K21_free_s1234_1925f3 cached (bits 4.1544)\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.1329)\n", " [4/4] single_m1_K64_free_s1234_194318 cached (bits 2.9487)\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=120 arms=2 (cached 2, fresh 0)\n", " [1/2] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.3582)\n", " [2/2] res_m3_K21_free_s1234_1925f3 cached (bits 4.4074)\n", "[push] dry run — skipped\n", "\n", "[rung 0] tier=P steps=60 arms=5 (cached 4, fresh 1)\n", " [1/5] sum_m3_K21_free_s1234_7faffe cached (bits 3.6814)\n", " [2/5] res_m3_K21_free_s1234_1925f3 cached (bits 4.1544)\n", " [3/5] prod_m3_K21_free_s1234_c5d5b7 cached (bits 4.1329)\n", " [4/5] gate_m3_K21_free_s1234_0d3f58 held 3.54 ce 4.379 dlv 1.62 [2.7871, 0.3954, 0.3608] acc 0.134 red 0.27\n", " [5/5] single_m1_K64_free_s1234_194318 cached (bits 2.9487)\n", "[push] dry run — skipped\n", " ✓ resume: full-cache fast path + mixed cache/fresh both work\n", "\n", "[rung 0] tier=P steps=40 arms=4 (cached 0, fresh 4)\n", " [1/4] sum_m3_K21_free_s1234_7faffe held 3.46 ce 5.407 dlv 0.59 [2.4185, 0.8562, 0.1877] acc 0.123 red 0.18\n", " [2/4] res_m3_K21_free_s1234_1925f3 held 3.72 ce 4.209 dlv 1.79 [2.8238, 0.6729, 0.2209] acc 0.203 red 0.11\n", " [3/4] prod_m3_K21_free_s1234_c5d5b7 held 3.67 ce 4.186 dlv 1.81 [2.6164, 0.6287, 0.4231] acc 0.221 red 0.07\n", " [4/4] single_m1_K64_free_s1234_194318 held 2.47 ce 5.279 dlv 0.72 [2.4703] acc 0.166 red 0.00\n", "[push] dry run — skipped\n", "\n", "[rung 1] tier=P steps=60 arms=2 (cached 0, fresh 2)\n", " [1/2] sum_m3_K21_free_s1234_7faffe held 3.38 ce 4.597 dlv 1.40 [2.5272, 0.7302, 0.1235] acc 0.135 red 0.22\n", " [2/2] res_m3_K21_free_s1234_1925f3 held 3.80 ce 3.152 dlv 2.85 [2.9599, 0.6163, 0.2213] acc 0.268 red 0.09\n", "[push] dry run — skipped\n", " ✓ protected controls ride the keep-line\n", "\n", "[rung 0] tier=L steps=25 arms=3 (cached 0, fresh 3)\n", " [1/3] prod_m2_K32_free_s1234_c2fd6a held 0.00 ce 5.619 dlv 0.00 [0.0, 0.0] acc 0.017 red 0.00 a 0.153\n", " [2/3] single_m1_K64_free_s1234_194318 held 0.00 ce 5.624 dlv -0.01 [0.0] acc 0.027 red 0.00 a 0.160\n", " [3/3] sum_m2_K32_free_s1234_6e47da held 0.00 ce 5.618 dlv -0.01 [0.0, 0.0] acc 0.023 red 0.01 a 0.154\n", "[push] dry run — skipped\n", " ✓ Tier-L rung: bpb finite, delivered logged, ledger schema intact\n", "[screen] nothing queued — ledger already holds 7 rows in exp011_smoke/results.csv; call report(fc) to view standings.\n", " ✓ empty-queue guard returns ledger standings\n", "acd_forge smoke: ALL GREEN — next: phase2_screen()\n" ] } ] }, { "cell_type": "code", "source": [ "phase4_screen()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "FoOheC0-93KC", "outputId": "d691d9b3-6d1f-40dd-fb49-3a385d7996f0" }, "execution_count": 14, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[forge] Phase 4: 14 routed arms (single+sum protected)\n", "[TrigramStream] loading HF corpus wikitext-103-raw-v1 ...\n", "[TrigramStream] 100,000,000 bytes = 33,333,333 trigrams\n", "\n", "[rung 0] tier=L steps=1000 arms=14 (cached 0, fresh 14)\n", " [1/14] single_m1_K64_routed_free_s1234_2b84f1 held 0.68 ce 3.285 dlv 1.28 [0.6813] acc 0.303 red 0.00 a 0.000\n", " [2/14] sum_m2_K64_routed_free_s1234_75e5dc held 0.92 ce 3.251 dlv 1.31 [0.8423, 0.0736] acc 0.328 red 0.03 a 0.000\n", " [3/14] sum_m4_K64_routed_free_s1234_da9153 held 0.81 ce 3.241 dlv 1.39 [0.6331, 0.1347, 0.0433, 0.0] acc 0.300 red 0.11 a 0.000\n", " [4/14] prod_m2_K64_routed_free_s1234_0c4a5c held 0.95 ce 3.235 dlv 1.34 [0.7024, 0.2469] acc 0.337 red 0.09 a 0.000\n", " [5/14] prod_m4_K64_routed_free_s1234_9d5fd9 held 1.16 ce 3.270 dlv 1.28 [0.6558, 0.1652, 0.2646, 0.077] acc 0.357 red 0.07 a 0.000\n", " [6/14] res_m2_K64_routed_free_s1234_fa9eba held 0.85 ce 3.255 dlv 1.37 [0.8194, 0.0313] acc 0.314 red 0.11 a 0.000\n", " [7/14] res_m4_K64_routed_free_s1234_33fa4e held 0.83 ce 3.249 dlv 1.36 [0.6654, 0.1616, 0.0, 0.0] acc 0.315 red 0.11 a 0.000\n", " [8/14] single_m1_K64_routed_free_s5678_fe6982 held 0.60 ce 3.255 dlv 1.36 [0.6001] acc 0.251 red 0.00 a 0.000\n", " [9/14] sum_m2_K64_routed_free_s5678_96435e held 0.47 ce 3.238 dlv 1.33 [0.417, 0.0515] acc 0.252 red 0.08 a 0.000\n", " [10/14] sum_m4_K64_routed_free_s5678_6eafa6 held 0.71 ce 3.263 dlv 1.34 [0.6104, 0.0073, 0.0907, 0.0] acc 0.283 red 0.12 a 0.000\n", " [11/14] prod_m2_K64_routed_free_s5678_f7c0b9 held 0.93 ce 3.255 dlv 1.30 [0.6985, 0.2272] acc 0.311 red 0.11 a 0.000\n", " [12/14] prod_m4_K64_routed_free_s5678_3da152 held 1.25 ce 3.242 dlv 1.41 [0.5856, 0.3634, 0.1937, 0.1042] acc 0.369 red 0.09 a 0.000\n", " [13/14] res_m2_K64_routed_free_s5678_795c93 held 0.75 ce 3.250 dlv 1.38 [0.6922, 0.0596] acc 0.283 red 0.09 a 0.000\n", " [14/14] res_m4_K64_routed_free_s5678_fd6144 held 0.65 ce 3.260 dlv 1.29 [0.5484, 0.019, 0.0546, 0.0268] acc 0.287 red 0.12 a 0.000\n", "[push] exp011R/ -> AbstractPhil/geolip-aleph-differentiation ✓\n", "\n", "[rung 1] tier=L steps=5000 arms=12 (cached 0, fresh 12)\n", " [1/12] sum_m4_K64_routed_free_s5678_6eafa6 held 0.26 ce 2.801 dlv 1.86 [0.1817, 0.0304, 0.0214, 0.0311] acc 0.206 red 0.13 a 0.000\n", " [2/12] res_m2_K64_routed_free_s1234_fa9eba held 0.31 ce 2.807 dlv 1.79 [0.2758, 0.0314] acc 0.260 red 0.03 a 0.000\n", " [3/12] single_m1_K64_routed_free_s1234_2b84f1 held 0.23 ce 2.807 dlv 1.80 [0.2271] acc 0.219 red 0.00 a 0.000\n", " [4/12] sum_m2_K64_routed_free_s5678_96435e held 0.10 ce 2.809 dlv 1.71 [0.0025, 0.1004] acc 0.186 red 0.19 a 0.000\n", " [5/12] single_m1_K64_routed_free_s5678_fe6982 held 0.07 ce 2.825 dlv 1.79 [0.0657] acc 0.191 red 0.00 a 0.000\n", " [6/12] res_m4_K64_routed_free_s1234_33fa4e held 0.15 ce 2.796 dlv 1.75 [0.1321, 0.0052, 0.0124, 0.0] acc 0.204 red 0.08 a 0.000\n", " [7/12] sum_m4_K64_routed_free_s1234_da9153 held 0.26 ce 2.801 dlv 1.77 [0.1975, 0.0508, 0.012, 0.0] acc 0.232 red 0.08 a 0.000\n", " [8/12] prod_m2_K64_routed_free_s1234_0c4a5c held 0.31 ce 2.806 dlv 1.77 [0.2434, 0.068] acc 0.232 red 0.14 a 0.000\n", " [9/12] prod_m4_K64_routed_free_s1234_9d5fd9 held 0.75 ce 2.893 dlv 1.75 [0.3159, 0.1637, 0.1008, 0.1668] acc 0.299 red 0.08 a 0.000\n", " [10/12] prod_m2_K64_routed_free_s5678_f7c0b9 held 0.44 ce 2.831 dlv 1.77 [0.2876, 0.1558] acc 0.255 red 0.06 a 0.000\n", " [11/12] sum_m2_K64_routed_free_s1234_75e5dc held 0.21 ce 2.802 dlv 1.86 [0.1748, 0.0384] acc 0.215 red 0.07 a 0.000\n", " [12/12] prod_m4_K64_routed_free_s5678_3da152 held 0.63 ce 2.905 dlv 1.70 [0.238, 0.1899, 0.0801, 0.1174] acc 0.273 red 0.05 a 0.000\n", "[push] exp011R/ -> AbstractPhil/geolip-aleph-differentiation ✓\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "[{'arm_id': 'single_m1_K64_routed_free_s1234_2b84f1',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6753858,\n", " 'acc': 0.3032,\n", " 'ce_bits': 3.2855,\n", " 'delivered_bits': 1.2773,\n", " 'cum_bits': 0.6813,\n", " 'marginal_bits': '[0.6813]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0052,\n", " 'rank_mean': 3.939,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 22.8,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m2_K64_routed_free_s1234_75e5dc',\n", " 'op': 'sum',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6753858,\n", " 'acc': 0.3281,\n", " 'ce_bits': 3.2508,\n", " 'delivered_bits': 1.3073,\n", " 'cum_bits': 0.9159,\n", " 'marginal_bits': '[0.8423, 0.0736]',\n", " 'redundancy': 0.0311,\n", " 'cancellation': 0.6685,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0033,\n", " 'rank_mean': 3.9,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 24.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'sum_m4_K64_routed_free_s1234_da9153',\n", " 'op': 'sum',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6753858,\n", " 'acc': 0.2998,\n", " 'ce_bits': 3.241,\n", " 'delivered_bits': 1.3884,\n", " 'cum_bits': 0.8111,\n", " 'marginal_bits': '[0.6331, 0.1347, 0.0433, 0.0]',\n", " 'redundancy': 0.1084,\n", " 'cancellation': 0.9314,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0076,\n", " 'rank_mean': 3.771,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 31.9,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'prod_m2_K64_routed_free_s1234_0c4a5c',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6827586,\n", " 'acc': 0.3372,\n", " 'ce_bits': 3.2346,\n", " 'delivered_bits': 1.3413,\n", " 'cum_bits': 0.9493,\n", " 'marginal_bits': '[0.7024, 0.2469]',\n", " 'redundancy': 0.0854,\n", " 'cancellation': 0.4275,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0036,\n", " 'rank_mean': 3.9,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 24.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'prod_m4_K64_routed_free_s1234_9d5fd9',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6975042,\n", " 'acc': 0.3569,\n", " 'ce_bits': 3.2696,\n", " 'delivered_bits': 1.2801,\n", " 'cum_bits': 1.1626,\n", " 'marginal_bits': '[0.6558, 0.1652, 0.2646, 0.077]',\n", " 'redundancy': 0.0688,\n", " 'cancellation': 0.9374,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0074,\n", " 'rank_mean': 3.773,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 33.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'res_m2_K64_routed_free_s1234_fa9eba',\n", " 'op': 'res',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6755906,\n", " 'acc': 0.3142,\n", " 'ce_bits': 3.2554,\n", " 'delivered_bits': 1.3713,\n", " 'cum_bits': 0.8507,\n", " 'marginal_bits': '[0.8194, 0.0313]',\n", " 'redundancy': 0.1105,\n", " 'cancellation': 0.5896,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0034,\n", " 'rank_mean': 3.9,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 25.6,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'res_m4_K64_routed_free_s1234_33fa4e',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6755906,\n", " 'acc': 0.3147,\n", " 'ce_bits': 3.2489,\n", " 'delivered_bits': 1.3647,\n", " 'cum_bits': 0.827,\n", " 'marginal_bits': '[0.6654, 0.1616, 0.0, 0.0]',\n", " 'redundancy': 0.1079,\n", " 'cancellation': 0.9082,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0078,\n", " 'rank_mean': 3.77,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 35.1,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_routed_free_s5678_fe6982',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6753858,\n", " 'acc': 0.2512,\n", " 'ce_bits': 3.2546,\n", " 'delivered_bits': 1.365,\n", " 'cum_bits': 0.6001,\n", " 'marginal_bits': '[0.6001]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0013,\n", " 'rank_mean': 3.947,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 22.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m2_K64_routed_free_s5678_96435e',\n", " 'op': 'sum',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6753858,\n", " 'acc': 0.252,\n", " 'ce_bits': 3.2381,\n", " 'delivered_bits': 1.33,\n", " 'cum_bits': 0.4685,\n", " 'marginal_bits': '[0.417, 0.0515]',\n", " 'redundancy': 0.0777,\n", " 'cancellation': 0.433,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0006,\n", " 'rank_mean': 3.903,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 24.2,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'sum_m4_K64_routed_free_s5678_6eafa6',\n", " 'op': 'sum',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6753858,\n", " 'acc': 0.2827,\n", " 'ce_bits': 3.2634,\n", " 'delivered_bits': 1.3367,\n", " 'cum_bits': 0.7083,\n", " 'marginal_bits': '[0.6104, 0.0073, 0.0907, 0.0]',\n", " 'redundancy': 0.1182,\n", " 'cancellation': 0.8652,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0064,\n", " 'rank_mean': 3.81,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 32.1,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'protected control'},\n", " {'arm_id': 'prod_m2_K64_routed_free_s5678_f7c0b9',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6827586,\n", " 'acc': 0.3108,\n", " 'ce_bits': 3.2548,\n", " 'delivered_bits': 1.3042,\n", " 'cum_bits': 0.9258,\n", " 'marginal_bits': '[0.6985, 0.2272]',\n", " 'redundancy': 0.1101,\n", " 'cancellation': 0.4437,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0006,\n", " 'rank_mean': 3.903,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 24.4,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'prod_m4_K64_routed_free_s5678_3da152',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6975042,\n", " 'acc': 0.3694,\n", " 'ce_bits': 3.2424,\n", " 'delivered_bits': 1.4137,\n", " 'cum_bits': 1.2469,\n", " 'marginal_bits': '[0.5856, 0.3634, 0.1937, 0.1042]',\n", " 'redundancy': 0.0855,\n", " 'cancellation': 0.9011,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0069,\n", " 'rank_mean': 3.813,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 33.5,\n", " 'rung': 0,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 8/14 by cum_bits'},\n", " {'arm_id': 'res_m2_K64_routed_free_s5678_795c93',\n", " 'op': 'res',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6755906,\n", " 'acc': 0.2832,\n", " 'ce_bits': 3.2503,\n", " 'delivered_bits': 1.3757,\n", " 'cum_bits': 0.7518,\n", " 'marginal_bits': '[0.6922, 0.0596]',\n", " 'redundancy': 0.0913,\n", " 'cancellation': 0.3614,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0006,\n", " 'rank_mean': 3.903,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 25.6,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'res_m4_K64_routed_free_s5678_fd6144',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 1000,\n", " 'params': 6755906,\n", " 'acc': 0.2866,\n", " 'ce_bits': 3.26,\n", " 'delivered_bits': 1.2865,\n", " 'cum_bits': 0.6488,\n", " 'marginal_bits': '[0.5484, 0.019, 0.0546, 0.0268]',\n", " 'redundancy': 0.1188,\n", " 'cancellation': 0.8108,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0065,\n", " 'rank_mean': 3.81,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 35.2,\n", " 'rung': 0,\n", " 'verdict': 'PARK',\n", " 'reason': 'below keep line'},\n", " {'arm_id': 'sum_m4_K64_routed_free_s5678_6eafa6',\n", " 'op': 'sum',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6753858,\n", " 'acc': 0.2061,\n", " 'ce_bits': 2.8012,\n", " 'delivered_bits': 1.8573,\n", " 'cum_bits': 0.2646,\n", " 'marginal_bits': '[0.1817, 0.0304, 0.0214, 0.0311]',\n", " 'redundancy': 0.1303,\n", " 'cancellation': 0.8026,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0123,\n", " 'rank_mean': 3.831,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 155.2,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'res_m2_K64_routed_free_s1234_fa9eba',\n", " 'op': 'res',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6755906,\n", " 'acc': 0.2595,\n", " 'ce_bits': 2.8071,\n", " 'delivered_bits': 1.7916,\n", " 'cum_bits': 0.3072,\n", " 'marginal_bits': '[0.2758, 0.0314]',\n", " 'redundancy': 0.0322,\n", " 'cancellation': 0.6663,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0005,\n", " 'rank_mean': 3.91,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 125.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_routed_free_s1234_2b84f1',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6753858,\n", " 'acc': 0.2192,\n", " 'ce_bits': 2.8071,\n", " 'delivered_bits': 1.8027,\n", " 'cum_bits': 0.2271,\n", " 'marginal_bits': '[0.2271]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0045,\n", " 'rank_mean': 3.94,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 109.2,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'sum_m2_K64_routed_free_s5678_96435e',\n", " 'op': 'sum',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6753858,\n", " 'acc': 0.1858,\n", " 'ce_bits': 2.8092,\n", " 'delivered_bits': 1.7095,\n", " 'cum_bits': 0.1029,\n", " 'marginal_bits': '[0.0025, 0.1004]',\n", " 'redundancy': 0.1941,\n", " 'cancellation': 0.1714,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0047,\n", " 'rank_mean': 3.916,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 118.3,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'single_m1_K64_routed_free_s5678_fe6982',\n", " 'op': 'single',\n", " 'm': 1,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6753858,\n", " 'acc': 0.1914,\n", " 'ce_bits': 2.8251,\n", " 'delivered_bits': 1.7916,\n", " 'cum_bits': 0.0657,\n", " 'marginal_bits': '[0.0657]',\n", " 'redundancy': 0.0,\n", " 'cancellation': 0.0,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0005,\n", " 'rank_mean': 3.95,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 109.1,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'res_m4_K64_routed_free_s1234_33fa4e',\n", " 'op': 'res',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6755906,\n", " 'acc': 0.2039,\n", " 'ce_bits': 2.7957,\n", " 'delivered_bits': 1.7521,\n", " 'cum_bits': 0.1497,\n", " 'marginal_bits': '[0.1321, 0.0052, 0.0124, 0.0]',\n", " 'redundancy': 0.0755,\n", " 'cancellation': 0.9543,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0023,\n", " 'rank_mean': 3.808,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 171.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'sum_m4_K64_routed_free_s1234_da9153',\n", " 'op': 'sum',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6753858,\n", " 'acc': 0.2317,\n", " 'ce_bits': 2.8011,\n", " 'delivered_bits': 1.7687,\n", " 'cum_bits': 0.2603,\n", " 'marginal_bits': '[0.1975, 0.0508, 0.012, 0.0]',\n", " 'redundancy': 0.0756,\n", " 'cancellation': 0.8955,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0024,\n", " 'rank_mean': 3.806,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 155.7,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'prod_m2_K64_routed_free_s1234_0c4a5c',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6827586,\n", " 'acc': 0.2322,\n", " 'ce_bits': 2.8064,\n", " 'delivered_bits': 1.7673,\n", " 'cum_bits': 0.3114,\n", " 'marginal_bits': '[0.2434, 0.068]',\n", " 'redundancy': 0.1435,\n", " 'cancellation': 0.4023,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0023,\n", " 'rank_mean': 3.903,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 119.4,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'prod_m4_K64_routed_free_s1234_9d5fd9',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6975042,\n", " 'acc': 0.2993,\n", " 'ce_bits': 2.8925,\n", " 'delivered_bits': 1.7515,\n", " 'cum_bits': 0.7472,\n", " 'marginal_bits': '[0.3159, 0.1637, 0.1008, 0.1668]',\n", " 'redundancy': 0.0823,\n", " 'cancellation': 0.815,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': -0.0044,\n", " 'rank_mean': 3.787,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 163.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'prod_m2_K64_routed_free_s5678_f7c0b9',\n", " 'op': 'prod',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6827586,\n", " 'acc': 0.2549,\n", " 'ce_bits': 2.8306,\n", " 'delivered_bits': 1.7697,\n", " 'cum_bits': 0.4434,\n", " 'marginal_bits': '[0.2876, 0.1558]',\n", " 'redundancy': 0.0551,\n", " 'cancellation': 0.5779,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0005,\n", " 'rank_mean': 3.904,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 119.4,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'sum_m2_K64_routed_free_s1234_75e5dc',\n", " 'op': 'sum',\n", " 'm': 2,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 1234,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6753858,\n", " 'acc': 0.2148,\n", " 'ce_bits': 2.8016,\n", " 'delivered_bits': 1.8596,\n", " 'cum_bits': 0.2132,\n", " 'marginal_bits': '[0.1748, 0.0384]',\n", " 'redundancy': 0.0737,\n", " 'cancellation': 0.6269,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.0022,\n", " 'rank_mean': 3.916,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 118.4,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'},\n", " {'arm_id': 'prod_m4_K64_routed_free_s5678_3da152',\n", " 'op': 'prod',\n", " 'm': 4,\n", " 'K': 64,\n", " 'd_addr': 4,\n", " 'freeze': 'free',\n", " 'seed': 5678,\n", " 'tree_hard': False,\n", " 'steps': 5000,\n", " 'params': 6975042,\n", " 'acc': 0.2732,\n", " 'ce_bits': 2.9052,\n", " 'delivered_bits': 1.7044,\n", " 'cum_bits': 0.6253,\n", " 'marginal_bits': '[0.238, 0.1899, 0.0801, 0.1174]',\n", " 'redundancy': 0.0537,\n", " 'cancellation': 0.949,\n", " 'stage_snr': 0.0,\n", " 'dev_mean': 0.009,\n", " 'rank_mean': 3.823,\n", " 'alpha': 0.0,\n", " 'killed': '',\n", " 'wall_s': 163.0,\n", " 'rung': 1,\n", " 'verdict': 'PROMOTE',\n", " 'reason': 'top 12/12 by cum_bits'}]" ] }, "metadata": {}, "execution_count": 14 } ] } ] }