Upload folder using huggingface_hub

AS-0.history.json

@@ -0,0 +1,107 @@
+[
+  {
+    "iter": 0,
+    "train": 4.539572238922119,
+    "val": 4.542599678039551
+  },
+  {
+    "iter": 250,
+    "train": 2.594716787338257,
+    "val": 2.589973211288452
+  },
+  {
+    "iter": 500,
+    "train": 2.3268213272094727,
+    "val": 2.3106179237365723
+  },
+  {
+    "iter": 750,
+    "train": 2.098438024520874,
+    "val": 2.0775511264801025
+  },
+  {
+    "iter": 1000,
+    "train": 1.9633289575576782,
+    "val": 1.9550288915634155
+  },
+  {
+    "iter": 1250,
+    "train": 1.8606176376342773,
+    "val": 1.8715767860412598
+  },
+  {
+    "iter": 1500,
+    "train": 1.7806389331817627,
+    "val": 1.8435417413711548
+  },
+  {
+    "iter": 1750,
+    "train": 1.767372488975525,
+    "val": 1.803306221961975
+  },
+  {
+    "iter": 2000,
+    "train": 1.7258944511413574,
+    "val": 1.7784298658370972
+  },
+  {
+    "iter": 2250,
+    "train": 1.6946111917495728,
+    "val": 1.7363885641098022
+  },
+  {
+    "iter": 2500,
+    "train": 1.6631735563278198,
+    "val": 1.7293020486831665
+  },
+  {
+    "iter": 2750,
+    "train": 1.647719144821167,
+    "val": 1.709837794303894
+  },
+  {
+    "iter": 3000,
+    "train": 1.61244797706604,
+    "val": 1.6988787651062012
+  },
+  {
+    "iter": 3250,
+    "train": 1.6222702264785767,
+    "val": 1.681233525276184
+  },
+  {
+    "iter": 3500,
+    "train": 1.5948697328567505,
+    "val": 1.6761611700057983
+  },
+  {
+    "iter": 3750,
+    "train": 1.573941946029663,
+    "val": 1.6592583656311035
+  },
+  {
+    "iter": 4000,
+    "train": 1.5794099569320679,
+    "val": 1.65290105342865
+  },
+  {
+    "iter": 4250,
+    "train": 1.5525082349777222,
+    "val": 1.642384648323059
+  },
+  {
+    "iter": 4500,
+    "train": 1.541032314300537,
+    "val": 1.6460506916046143
+  },
+  {
+    "iter": 4750,
+    "train": 1.5588961839675903,
+    "val": 1.6441395282745361
+  },
+  {
+    "iter": 5000,
+    "train": 1.5567485094070435,
+    "val": 1.6493732929229736
+  }
+]

AS-0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c0a7f948560fd6d771c7ec22b739978d8471e2177584b2f410cd5812728cab
+size 3271279

AS-1.history.json

@@ -0,0 +1,67 @@
+[
+  {
+    "iter": 0,
+    "train": 4.552076816558838,
+    "val": 4.549188613891602
+  },
+  {
+    "iter": 250,
+    "train": 2.629171848297119,
+    "val": 2.6566600799560547
+  },
+  {
+    "iter": 500,
+    "train": 2.3648459911346436,
+    "val": 2.372241735458374
+  },
+  {
+    "iter": 750,
+    "train": 2.1083717346191406,
+    "val": 2.1116890907287598
+  },
+  {
+    "iter": 1000,
+    "train": 1.9503668546676636,
+    "val": 1.9873727560043335
+  },
+  {
+    "iter": 1250,
+    "train": 1.8487746715545654,
+    "val": 1.8949698209762573
+  },
+  {
+    "iter": 1500,
+    "train": 1.787669062614441,
+    "val": 1.8391720056533813
+  },
+  {
+    "iter": 1750,
+    "train": 1.7346687316894531,
+    "val": 1.7939311265945435
+  },
+  {
+    "iter": 2000,
+    "train": 1.7033478021621704,
+    "val": 1.7674237489700317
+  },
+  {
+    "iter": 2250,
+    "train": 1.6747982501983643,
+    "val": 1.747664451599121
+  },
+  {
+    "iter": 2500,
+    "train": 1.6424914598464966,
+    "val": 1.7313590049743652
+  },
+  {
+    "iter": 2750,
+    "train": 1.6468055248260498,
+    "val": 1.717896580696106
+  },
+  {
+    "iter": 3000,
+    "train": 1.6467840671539307,
+    "val": 1.7149957418441772
+  }
+]

AS-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b72c9258ec0a039269ca7ba4f4c352a60922c6ae8ffbbedfb9367d43639e8b8
+size 3290255

AS-2.history.json

@@ -0,0 +1,67 @@
+[
+  {
+    "iter": 0,
+    "train": 4.5503106117248535,
+    "val": 4.547379016876221
+  },
+  {
+    "iter": 250,
+    "train": 2.6311113834381104,
+    "val": 2.6660752296447754
+  },
+  {
+    "iter": 500,
+    "train": 2.3749659061431885,
+    "val": 2.3779335021972656
+  },
+  {
+    "iter": 750,
+    "train": 2.127347946166992,
+    "val": 2.1233832836151123
+  },
+  {
+    "iter": 1000,
+    "train": 1.9767907857894897,
+    "val": 2.000434398651123
+  },
+  {
+    "iter": 1250,
+    "train": 1.873072862625122,
+    "val": 1.918302059173584
+  },
+  {
+    "iter": 1500,
+    "train": 1.8198323249816895,
+    "val": 1.8644909858703613
+  },
+  {
+    "iter": 1750,
+    "train": 1.7653547525405884,
+    "val": 1.8185434341430664
+  },
+  {
+    "iter": 2000,
+    "train": 1.734865427017212,
+    "val": 1.7887829542160034
+  },
+  {
+    "iter": 2250,
+    "train": 1.7053745985031128,
+    "val": 1.7657300233840942
+  },
+  {
+    "iter": 2500,
+    "train": 1.6744619607925415,
+    "val": 1.760308027267456
+  },
+  {
+    "iter": 2750,
+    "train": 1.6789984703063965,
+    "val": 1.7330682277679443
+  },
+  {
+    "iter": 3000,
+    "train": 1.6782021522521973,
+    "val": 1.7376532554626465
+  }
+]

AS-2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd8509c033807624724cabcb7a97ae02b7d30b287858bc9c34e12c89022d70d8
+size 3290255

AS-3.history.json

@@ -0,0 +1,67 @@
+[
+  {
+    "iter": 0,
+    "train": 4.5514044761657715,
+    "val": 4.5474395751953125
+  },
+  {
+    "iter": 250,
+    "train": 2.6267995834350586,
+    "val": 2.672171115875244
+  },
+  {
+    "iter": 500,
+    "train": 2.363349199295044,
+    "val": 2.3642754554748535
+  },
+  {
+    "iter": 750,
+    "train": 2.1582014560699463,
+    "val": 2.171232223510742
+  },
+  {
+    "iter": 1000,
+    "train": 2.0154964923858643,
+    "val": 2.0406768321990967
+  },
+  {
+    "iter": 1250,
+    "train": 1.90642511844635,
+    "val": 1.938228726387024
+  },
+  {
+    "iter": 1500,
+    "train": 1.8478271961212158,
+    "val": 1.8756272792816162
+  },
+  {
+    "iter": 1750,
+    "train": 1.7902661561965942,
+    "val": 1.8274116516113281
+  },
+  {
+    "iter": 2000,
+    "train": 1.7633624076843262,
+    "val": 1.8050482273101807
+  },
+  {
+    "iter": 2250,
+    "train": 1.7267961502075195,
+    "val": 1.7800605297088623
+  },
+  {
+    "iter": 2500,
+    "train": 1.6958733797073364,
+    "val": 1.7739835977554321
+  },
+  {
+    "iter": 2750,
+    "train": 1.6959929466247559,
+    "val": 1.7510572671890259
+  },
+  {
+    "iter": 3000,
+    "train": 1.697333574295044,
+    "val": 1.7450127601623535
+  }
+]

AS-3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b197578a909e2a3b51e34604152b063ee6359bd0e568cf1568fd90a09033aea5
+size 3290255

AS-4.history.json

@@ -0,0 +1,107 @@
+[
+  {
+    "iter": 0,
+    "train": 4.548327445983887,
+    "val": 4.553555488586426
+  },
+  {
+    "iter": 250,
+    "train": 2.6702470779418945,
+    "val": 2.7029836177825928
+  },
+  {
+    "iter": 500,
+    "train": 2.4296441078186035,
+    "val": 2.4365997314453125
+  },
+  {
+    "iter": 750,
+    "train": 2.1754744052886963,
+    "val": 2.148674964904785
+  },
+  {
+    "iter": 1000,
+    "train": 2.030128240585327,
+    "val": 1.9878582954406738
+  },
+  {
+    "iter": 1250,
+    "train": 1.9509085416793823,
+    "val": 1.932215929031372
+  },
+  {
+    "iter": 1500,
+    "train": 1.8665056228637695,
+    "val": 1.9155062437057495
+  },
+  {
+    "iter": 1750,
+    "train": 1.8813914060592651,
+    "val": 1.8996834754943848
+  },
+  {
+    "iter": 2000,
+    "train": 1.8438420295715332,
+    "val": 1.866635799407959
+  },
+  {
+    "iter": 2250,
+    "train": 1.8256421089172363,
+    "val": 1.8442517518997192
+  },
+  {
+    "iter": 2500,
+    "train": 1.8107391595840454,
+    "val": 1.8394869565963745
+  },
+  {
+    "iter": 2750,
+    "train": 1.7852716445922852,
+    "val": 1.8160736560821533
+  },
+  {
+    "iter": 3000,
+    "train": 1.7610642910003662,
+    "val": 1.8150714635849
+  },
+  {
+    "iter": 3250,
+    "train": 1.7708102464675903,
+    "val": 1.7933977842330933
+  },
+  {
+    "iter": 3500,
+    "train": 1.7449135780334473,
+    "val": 1.7923871278762817
+  },
+  {
+    "iter": 3750,
+    "train": 1.729687213897705,
+    "val": 1.7667193412780762
+  },
+  {
+    "iter": 4000,
+    "train": 1.727928876876831,
+    "val": 1.7680315971374512
+  },
+  {
+    "iter": 4250,
+    "train": 1.6951998472213745,
+    "val": 1.756170392036438
+  },
+  {
+    "iter": 4500,
+    "train": 1.6917592287063599,
+    "val": 1.7556484937667847
+  },
+  {
+    "iter": 4750,
+    "train": 1.7072231769561768,
+    "val": 1.7557003498077393
+  },
+  {
+    "iter": 5000,
+    "train": 1.7068597078323364,
+    "val": 1.7644997835159302
+  }
+]

AS-4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b58d85acc1ac44414b0b8090d0a2d7acd38f38f22d746a94e4693b28e0ff7057
+size 3290255

AS-5.history.json

@@ -0,0 +1,67 @@
+[
+  {
+    "iter": 0,
+    "train": 4.55631160736084,
+    "val": 4.5502028465271
+  },
+  {
+    "iter": 250,
+    "train": 2.681396484375,
+    "val": 2.707669258117676
+  },
+  {
+    "iter": 500,
+    "train": 2.542595863342285,
+    "val": 2.5977771282196045
+  },
+  {
+    "iter": 750,
+    "train": 2.359412908554077,
+    "val": 2.3929383754730225
+  },
+  {
+    "iter": 1000,
+    "train": 2.241136074066162,
+    "val": 2.2486231327056885
+  },
+  {
+    "iter": 1250,
+    "train": 2.1591928005218506,
+    "val": 2.13960599899292
+  },
+  {
+    "iter": 1500,
+    "train": 2.0773978233337402,
+    "val": 2.0679638385772705
+  },
+  {
+    "iter": 1750,
+    "train": 2.035414218902588,
+    "val": 2.0333921909332275
+  },
+  {
+    "iter": 2000,
+    "train": 1.997584581375122,
+    "val": 1.9956610202789307
+  },
+  {
+    "iter": 2250,
+    "train": 1.9860050678253174,
+    "val": 1.9897557497024536
+  },
+  {
+    "iter": 2500,
+    "train": 1.9610165357589722,
+    "val": 1.9707989692687988
+  },
+  {
+    "iter": 2750,
+    "train": 1.9531865119934082,
+    "val": 1.9446682929992676
+  },
+  {
+    "iter": 3000,
+    "train": 1.967576265335083,
+    "val": 1.95204496383667
+  }
+]

AS-5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58c8402f862a5ce878affb15320c06f11467781a6139bd4359f94f23c555ff9a
+size 1432587

README.md

@@ -0,0 +1,93 @@
+---
+license: mit
+language: [en]
+tags: [text-generation, quantization, bitnet, tiny, from-scratch]
+pipeline_tag: text-generation
+---
+
+# Artificial Stupidity — the tiny models (AS-0 … AS-5)
+
+Six character-level language models **built from scratch** — no GPT-2, no
+pretrained anything — that are identical in every respect except **how many
+bits each weight is allowed**. The smallest is **83 KB**.
+
+They are the control experiment for
+[ayushmaninbox/artificial-stupidity](https://huggingface.co/ayushmaninbox/artificial-stupidity),
+which is fluent because OpenAI paid for the fluency. These are not fluent, and
+every parameter is homegrown.
+
+## The six
+
+| | Weight precision | Params | Packed size | Val loss |
+|---|---|--:|--:|--:|
+| **AS-0** | 32-bit (control) | 815,488 | 3.1 MB | 1.642 |
+| **AS-1** | 8-bit | 819,072 | 835 KB | — |
+| **AS-2** | 4-bit | 819,072 | 448 KB | — |
+| **AS-3** | 1.58-bit `{-1,0,+1}` | 819,072 | 216 KB | — |
+| **AS-4** | 1-bit `{-1,+1}` | 819,072 | **169 KB** | 1.756 |
+| **AS-5** | 1-bit + 8-bit activations | 350,784 | **83 KB** | — |
+
+## What they actually produce
+
+Real output, same prompt, same seed (`python compare.py --seed 1337`):
+
+**`why is the sky blue`**
+
+| Model | Size | Answer |
+|---|--:|---|
+| AS-0 | 3.1 MB | *ask ays what is 17* |
+| AS-1 | 835 KB | *no it isnt* |
+| AS-2 | 448 KB | *a lot* |
+| AS-3 | 216 KB | *ok* |
+| AS-4 | 169 KB | *no it is rain* |
+| AS-5 | 83 KB | *no 11 whats 4* |
+
+**`what is 2 + 2`** — every one of them answers **`127`** (AS-0 says `1`).
+They learned the *shape* of an arithmetic reply from the synthetic corpus
+without learning arithmetic.
+
+**`what is your name`**
+
+| Model | Answer |
+|---|---|
+| AS-0 | *definitely not* |
+| AS-2 | *definitely not ryand* |
+| AS-4 | *dog* |
+| AS-5 | *do 1542 plus 40 13* |
+
+They hold a conversational **register** — correctly spelled, real chat moves —
+while being completely disconnected from the question. AS-5 at 83 KB is where
+it visibly breaks: it keeps the rhythm of internet chat but can no longer stay
+on one topic for a whole sentence.
+
+## Quantization-aware, not post-hoc
+
+You **cannot** train a normal model and round its weights to 1 bit afterwards —
+you get static, and worse, you cannot tell "compression worked" from "my code is
+broken". These models know they are being squashed *while they learn*, via a
+straight-through estimator, so they route around the damage.
+
+1 bit is not a 32× saving but about **20×**: embeddings, LayerNorms and one fp16
+scale per weight row stay high-precision. BitNet keeps them too.
+
+## Limitations
+
+- **Character-level**, so they cannot spell reliably — "mitochondria" is twelve
+  consecutive guesses and they lose that bet.
+- Trained on 9.8 MB of Twitch chat, Reddit, YouTube transcripts and lyrics.
+- They are **not** useful. They are a measurement of where language breaks.
+
+## Usage
+
+```bash
+git clone https://github.com/ayushmaninbox/artificial-stupidity
+cd artificial-stupidity/as-text-model
+pip install -r requirements.txt
+python generate.py AS-4 --prompt "hello"
+python compare.py          # all six, side by side
+```
+
+## License
+
+MIT for the code and weights. The training corpus is scraped third-party text —
+see the [dataset card](https://huggingface.co/datasets/ayushmaninbox/artificial-stupidity-corpus).

tokenizer.json

@@ -0,0 +1 @@
+{"chars": ["\n", " ", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "@", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "]", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "😂", "😭"]}