nmarshall commited on
Commit
4d5abdd
·
1 Parent(s): d7eb276

storage: configure git and xet

Browse files
.gitattributes CHANGED
@@ -1,11 +1,107 @@
1
- testdata/** filter=lfs diff=lfs merge=lfs -text
2
- crates/**/tests/fixtures/** filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.gif filter=lfs diff=lfs merge=lfs -text
5
- *.wsc filter=lfs diff=lfs merge=lfs -text
6
- *.pal filter=lfs diff=lfs merge=lfs -text
7
- *.wagame filter=lfs diff=lfs merge=lfs -text
8
- *.WAgame filter=lfs diff=lfs merge=lfs -text
9
- crates/*/data/** filter=lfs diff=lfs merge=lfs -text
10
- testdata/**/*.md !text -filter -merge -diff
11
- testdata/**/.gitignore !text -filter -merge -diff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Code, documentation and manifests remain ordinary Git text.
2
+ * text=auto eol=lf filter=openwa-code
3
+ # Payload directories include small textual test data and reference files.
4
+ **/assets/** filter=lfs diff=lfs merge=lfs -text !eol
5
+ **/references/** filter=lfs diff=lfs merge=lfs -text !eol
6
+ **/assets-src/** filter=lfs diff=lfs merge=lfs -text !eol
7
+ **/compiled-assets/** filter=lfs diff=lfs merge=lfs -text !eol
8
+ **/compiled-shaders/** filter=lfs diff=lfs merge=lfs -text !eol
9
+ testdata/** filter=lfs diff=lfs merge=lfs -text !eol
10
+ crates/**/tests/fixtures/** filter=lfs diff=lfs merge=lfs -text !eol
11
+ crates/*/data/** filter=lfs diff=lfs merge=lfs -text !eol
12
+ # Payload formats outside these directories are also stored through Xet.
13
+ *.7[zZ] filter=lfs diff=lfs merge=lfs -text !eol
14
+ *.[aA] filter=lfs diff=lfs merge=lfs -text !eol
15
+ *.[aA][aA][sS] filter=lfs diff=lfs merge=lfs -text !eol
16
+ *.[aA][rR][eE][nN][aA] filter=lfs diff=lfs merge=lfs -text !eol
17
+ *.[aA][sS][eE] filter=lfs diff=lfs merge=lfs -text !eol
18
+ *.[bB][aA][nN][kK] filter=lfs diff=lfs merge=lfs -text !eol
19
+ *.[bB][iI][nN] filter=lfs diff=lfs merge=lfs -text !eol
20
+ *.[bB][lL][eE][nN][dD] filter=lfs diff=lfs merge=lfs -text !eol
21
+ *.[bB][mM][pP] filter=lfs diff=lfs merge=lfs -text !eol
22
+ *.[bB][oO][tT] filter=lfs diff=lfs merge=lfs -text !eol
23
+ *.[bB][sS][pP] filter=lfs diff=lfs merge=lfs -text !eol
24
+ *.[cC][rR][oO][sS][sS][hH][aA][iI][rR] filter=lfs diff=lfs merge=lfs -text !eol
25
+ *.[cC][uU][bB][eE] filter=lfs diff=lfs merge=lfs -text !eol
26
+ *.[dD][aA][tT] filter=lfs diff=lfs merge=lfs -text !eol
27
+ *.[dD][dD][sS] filter=lfs diff=lfs merge=lfs -text !eol
28
+ *.[dD][lL][lL] filter=lfs diff=lfs merge=lfs -text !eol
29
+ *.[dD][oO][cC] filter=lfs diff=lfs merge=lfs -text !eol
30
+ *.[dD][oO][cC][xX] filter=lfs diff=lfs merge=lfs -text !eol
31
+ *.[dD][yY][lL][iI][bB] filter=lfs diff=lfs merge=lfs -text !eol
32
+ *.[eE][xX][eE] filter=lfs diff=lfs merge=lfs -text !eol
33
+ *.[eE][xX][rR] filter=lfs diff=lfs merge=lfs -text !eol
34
+ *.[fF][bB][xX] filter=lfs diff=lfs merge=lfs -text !eol
35
+ *.[fF][lL][aA][cC] filter=lfs diff=lfs merge=lfs -text !eol
36
+ *.[gG][iI][fF] filter=lfs diff=lfs merge=lfs -text !eol
37
+ *.[gG][lL][bB] filter=lfs diff=lfs merge=lfs -text !eol
38
+ *.[gG][lL][tT][fF] filter=lfs diff=lfs merge=lfs -text !eol
39
+ *.[gG][zZ] filter=lfs diff=lfs merge=lfs -text !eol
40
+ *.[hH][dD][rR] filter=lfs diff=lfs merge=lfs -text !eol
41
+ *.[iI][cC][nN][sS] filter=lfs diff=lfs merge=lfs -text !eol
42
+ *.[iI][cC][oO] filter=lfs diff=lfs merge=lfs -text !eol
43
+ *.[iI][qQ][mM] filter=lfs diff=lfs merge=lfs -text !eol
44
+ *.[iI][xX][mM] filter=lfs diff=lfs merge=lfs -text !eol
45
+ *.[iI][xX][pP][lL] filter=lfs diff=lfs merge=lfs -text !eol
46
+ *.[jJ][pP][eE][gG] filter=lfs diff=lfs merge=lfs -text !eol
47
+ *.[jJ][pP][gG] filter=lfs diff=lfs merge=lfs -text !eol
48
+ *.[kK][tT][xX] filter=lfs diff=lfs merge=lfs -text !eol
49
+ *.[kK][tT][xX]2 filter=lfs diff=lfs merge=lfs -text !eol
50
+ *.[lL][iI][bB] filter=lfs diff=lfs merge=lfs -text !eol
51
+ *.[mM][aA][pP] filter=lfs diff=lfs merge=lfs -text !eol
52
+ *.[mM][bB]3[dD][aA][tT] filter=lfs diff=lfs merge=lfs -text !eol
53
+ *.[mM][bB]3[tT][xX][tT] filter=lfs diff=lfs merge=lfs -text !eol
54
+ *.[mM][dD]3 filter=lfs diff=lfs merge=lfs -text !eol
55
+ *.[mM][dD]4 filter=lfs diff=lfs merge=lfs -text !eol
56
+ *.[mM][dD]5[aA][nN][iI][mM] filter=lfs diff=lfs merge=lfs -text !eol
57
+ *.[mM][dD]5[mM][eE][sS][hH] filter=lfs diff=lfs merge=lfs -text !eol
58
+ *.[mM][dD][cC] filter=lfs diff=lfs merge=lfs -text !eol
59
+ *.[mM][dD][sS] filter=lfs diff=lfs merge=lfs -text !eol
60
+ *.[mM][eE][sS][hH] filter=lfs diff=lfs merge=lfs -text !eol
61
+ *.[mM][oO][vV] filter=lfs diff=lfs merge=lfs -text !eol
62
+ *.[mM][pP]3 filter=lfs diff=lfs merge=lfs -text !eol
63
+ *.[mM][pP]4 filter=lfs diff=lfs merge=lfs -text !eol
64
+ *.[oO][bB][jJ] filter=lfs diff=lfs merge=lfs -text !eol
65
+ *.[oO][gG][gG] filter=lfs diff=lfs merge=lfs -text !eol
66
+ *.[oO][tT][fF] filter=lfs diff=lfs merge=lfs -text !eol
67
+ *.[pP][aA][lL] filter=lfs diff=lfs merge=lfs -text !eol
68
+ *.[pP][dD][bB] filter=lfs diff=lfs merge=lfs -text !eol
69
+ *.[pP][dD][fF] filter=lfs diff=lfs merge=lfs -text !eol
70
+ *.[pP][kK]3 filter=lfs diff=lfs merge=lfs -text !eol
71
+ *.[pP][lL][uU][gG][iI][nN] filter=lfs diff=lfs merge=lfs -text !eol
72
+ *.[pP][nN][gG] filter=lfs diff=lfs merge=lfs -text !eol
73
+ *.[pP][pP][tT][xX] filter=lfs diff=lfs merge=lfs -text !eol
74
+ *.[pP][rR][tT] filter=lfs diff=lfs merge=lfs -text !eol
75
+ *.[qQ][vV][mM] filter=lfs diff=lfs merge=lfs -text !eol
76
+ *.[rR][aA][dD] filter=lfs diff=lfs merge=lfs -text !eol
77
+ *.[rR][aA][rR] filter=lfs diff=lfs merge=lfs -text !eol
78
+ *.[rR][oO][qQ] filter=lfs diff=lfs merge=lfs -text !eol
79
+ *.[sS][hH][aA][dD][eE][rR] filter=lfs diff=lfs merge=lfs -text !eol
80
+ *.[sS][kK][iI][nN] filter=lfs diff=lfs merge=lfs -text !eol
81
+ *.[sS][nN][aA][pP][sS][hH][oO][tT] filter=lfs diff=lfs merge=lfs -text !eol
82
+ *.[sS][oO] filter=lfs diff=lfs merge=lfs -text !eol
83
+ *.[sS][pP][vV] filter=lfs diff=lfs merge=lfs -text !eol
84
+ *.[sS][rR][fF] filter=lfs diff=lfs merge=lfs -text !eol
85
+ *.[sS][vV][gG] filter=lfs diff=lfs merge=lfs -text !eol
86
+ *.[tT][aA][rR] filter=lfs diff=lfs merge=lfs -text !eol
87
+ *.[tT][eE][xX][tT][uU][rR][eE] filter=lfs diff=lfs merge=lfs -text !eol
88
+ *.[tT][gG][aA] filter=lfs diff=lfs merge=lfs -text !eol
89
+ *.[tT][iI][fF] filter=lfs diff=lfs merge=lfs -text !eol
90
+ *.[tT][iI][fF][fF] filter=lfs diff=lfs merge=lfs -text !eol
91
+ *.[tT][tT][fF] filter=lfs diff=lfs merge=lfs -text !eol
92
+ *.[vV][oO][iI][cC][eE] filter=lfs diff=lfs merge=lfs -text !eol
93
+ *.[wW][aA][gG][aA][mM][eE] filter=lfs diff=lfs merge=lfs -text !eol
94
+ *.[wW][aA][vV] filter=lfs diff=lfs merge=lfs -text !eol
95
+ *.[wW][eE][bB][mM] filter=lfs diff=lfs merge=lfs -text !eol
96
+ *.[wW][eE][bB][pP] filter=lfs diff=lfs merge=lfs -text !eol
97
+ *.[wW][oO][fF][fF] filter=lfs diff=lfs merge=lfs -text !eol
98
+ *.[wW][oO][fF][fF]2 filter=lfs diff=lfs merge=lfs -text !eol
99
+ *.[wW][sS][cC] filter=lfs diff=lfs merge=lfs -text !eol
100
+ *.[xX][lL][sS][xX] filter=lfs diff=lfs merge=lfs -text !eol
101
+ *.[xX][zZ] filter=lfs diff=lfs merge=lfs -text !eol
102
+ *.[zZ][iI][pP] filter=lfs diff=lfs merge=lfs -text !eol
103
+ *.[zZ][sS][tT] filter=lfs diff=lfs merge=lfs -text !eol
104
+ *.so.* filter=lfs diff=lfs merge=lfs -text !eol
105
+ # Test corpus documentation and ignore rules remain readable Git text.
106
+ testdata/**/*.md text=auto eol=lf filter=openwa-code !diff !merge
107
+ testdata/**/.gitignore text=auto eol=lf filter=openwa-code !diff !merge
.githooks/pre-commit ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ #!/bin/sh
2
+ set -eu
3
+ exec python3 scripts/storage_policy.py check --require-setup
.githooks/pre-push ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+ set -eu
3
+ updates=$(mktemp)
4
+ trap 'rm -f "$updates"' EXIT HUP INT TERM
5
+ cat > "$updates"
6
+ python3 scripts/storage_policy.py check --require-setup
7
+ python3 scripts/storage_policy.py push --require-setup < "$updates"
8
+ git lfs pre-push "$@" < "$updates"
README.md CHANGED
@@ -133,3 +133,11 @@ But it is true that this is one of the weirdest Rust codebases I've ever worked
133
  ## Disclaimer
134
 
135
  "Worms Armageddon" is a registered trademark of Team17 Software Limited in the EU and other countries. OpenWA is an independent and strictly non-commercial fan project, and is not affiliated with or endorsed by Team17 Software Limited. OpenWA does not include any of the original game's assets, and requires a legally acquired copy of "Worms Armageddon" to use.
 
 
 
 
 
 
 
 
 
133
  ## Disclaimer
134
 
135
  "Worms Armageddon" is a registered trademark of Team17 Software Limited in the EU and other countries. OpenWA is an independent and strictly non-commercial fan project, and is not affiliated with or endorsed by Team17 Software Limited. OpenWA does not include any of the original game's assets, and requires a legally acquired copy of "Worms Armageddon" to use.
136
+
137
+ ## Hugging Face checkout storage
138
+
139
+ This fork keeps source code in ordinary Git and test/reference payloads in
140
+ Git LFS backed by Xet at [nmarshall/wormz-a](https://huggingface.co/nmarshall/wormz-a).
141
+ After cloning, run `python3 scripts/setup-repository.py` to install the required
142
+ filters and hooks. See [Git and Xet storage](docs/storage.md) for setup, checks
143
+ and the relationship between this fork's history and upstream.
docs/storage.md ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Git and Xet storage
2
+
3
+ The Hugging Face remote is `git@hf.co:nmarshall/wormz-a` (`hf`).
4
+ The `main` branch contains the OpenWA master history with test payloads converted
5
+ to Git LFS pointers. Original authorship and commit messages are preserved, but
6
+ migration changes commit IDs. The original upstream history remains on local
7
+ `master` and `origin/*`; publish `main` to `hf`, not those original refs.
8
+
9
+ Source, scripts, manifests and documentation remain ordinary Git text.
10
+ Test replays, expected logs, snapshots, scheme fixtures, images and other assets
11
+ use Git LFS backed by Xet, including small textual test payloads.
12
+ `.gitattributes` declares this policy; `scripts/storage_policy.py` enforces it.
13
+ No original game installation or proprietary game assets are added by this setup.
14
+
15
+ ## Set up a checkout
16
+
17
+ Install Git, Python 3, Git LFS and Git Xet (the local Nix environment already
18
+ provides Git LFS and Git Xet), then run from the repository root:
19
+
20
+ ```sh
21
+ python3 scripts/setup-repository.py
22
+ git lfs pull hf
23
+ python3 scripts/storage_policy.py check --require-setup
24
+ ```
25
+
26
+ For a fresh clone, use:
27
+
28
+ ```sh
29
+ git clone -o hf git@hf.co:nmarshall/wormz-a
30
+ cd wormz-a
31
+ python3 scripts/setup-repository.py
32
+ ```
33
+
34
+ Setup is idempotent. It registers the required `openwa-code` clean filter,
35
+ Git LFS, the Xet custom transfer agent and `.githooks`. It refuses to replace
36
+ an existing local hooks path. The clean filter passes code through unchanged
37
+ and rejects misplaced payloads, non-UTF-8/binary content and code over 2 MiB.
38
+ Asset directory rules route even small text payloads through LFS. Documentation
39
+ and `.gitignore` files in `testdata/` are explicit ordinary-Git exceptions.
40
+
41
+ Commit and push normally:
42
+
43
+ ```sh
44
+ git add <paths>
45
+ git commit -m "area: short description"
46
+ git push hf main
47
+ ```
48
+
49
+ The pre-commit hook validates staged Git blobs and attributes. The pre-push hook
50
+ validates newly reachable history and then runs `git lfs pre-push`; Git LFS uses
51
+ its registered Xet transfer agent when negotiated with Hugging Face. Files remain
52
+ fully materialized in the working tree, while Git stores only asset pointers.
53
+ Do not bypass the hooks. If adding a new payload format or directory, update both
54
+ `.gitattributes` and the classifier in `scripts/storage_policy.py`.
55
+
56
+ Adding attributes affects future staging, not existing history. Do not merge the
57
+ unmigrated upstream refs into `main`: that would reintroduce raw historical
58
+ payloads. Future upstream synchronization needs the corresponding LFS conversion
59
+ before integration. Do not rewrite published history or force-push implicitly.
scripts/setup-repository.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Install the committed storage policy in this checkout's local Git config."""
3
+ from pathlib import Path
4
+ import shlex
5
+ import subprocess
6
+ import sys
7
+
8
+ ROOT = Path(__file__).resolve().parents[1]
9
+
10
+
11
+ def main():
12
+ if not (ROOT / ".git").exists():
13
+ raise SystemExit("Run setup in a Git checkout, not a Nix store source copy.")
14
+ def git(*args):
15
+ return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip()
16
+ existing = subprocess.run(["git", "-C", str(ROOT), "config", "--local", "--get", "core.hooksPath"],
17
+ text=True, capture_output=True).stdout.strip()
18
+ if existing and existing != ".githooks":
19
+ raise SystemExit(f"Existing hooksPath {existing!r}: integrate it before replacing hooks.")
20
+ git("lfs", "version")
21
+ git("xet", "--version")
22
+ git("lfs", "install", "--local", "--skip-repo")
23
+ git("xet", "install", "--local")
24
+ git("config", "--local", "lfs.basictransfersonly", "false")
25
+ python = shlex.quote(sys.executable)
26
+ script = shlex.quote(str(ROOT / "scripts/storage_policy.py"))
27
+ # Git shell-quotes its %f substitution. Keep it as a distinct argument.
28
+ git("config", "--local", "filter.openwa-code.clean", f"{python} {script} clean -- %f")
29
+ git("config", "--local", "filter.openwa-code.smudge", "cat")
30
+ git("config", "--local", "filter.openwa-code.required", "true")
31
+ git("config", "--local", "core.hooksPath", ".githooks")
32
+ for name in ("pre-commit", "pre-push"):
33
+ path = ROOT / ".githooks" / name
34
+ path.chmod(path.stat().st_mode | 0o111)
35
+ print("Installed local code/LFS filters, Xet transfers and commit/push guards.")
36
+
37
+
38
+ if __name__ == "__main__":
39
+ main()
scripts/storage_policy.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Fail-closed checks for code Git objects and asset LFS pointers."""
3
+ import argparse
4
+ from contextlib import contextmanager
5
+ import os
6
+ from pathlib import Path, PurePosixPath
7
+ import re
8
+ import subprocess
9
+ import sys
10
+ import shutil
11
+
12
+ MAX_CODE_BYTES = 2 * 1024 * 1024
13
+ PAYLOAD_SUFFIXES = frozenset(
14
+ ".png .jpg .jpeg .webp .gif .bmp .tga .tif .tiff .exr .hdr .svg .ico "
15
+ ".dds .ktx .ktx2 .texture .glb .gltf .obj .fbx .blend .mesh "
16
+ ".wav .ogg .mp3 .flac .bank .mp4 .webm .mov .ttf .otf .woff .woff2 "
17
+ ".exe .dll .pdb .so .dylib .a .lib .bin .spv .snapshot .cube "
18
+ ".zip .7z .rar .tar .gz .xz .zst .pdf .doc .docx .pptx .xlsx "
19
+ ".pk3 .bsp .aas .md3 .md4 .mdc .mds .md5mesh .md5anim .iqm .ase "
20
+ ".wagame .wsc .pal .ixpl .plugin .ixm .mb3dat .mb3txt .qvm .roq .icns .dat .map .prt .srf .rad .skin .shader .arena .bot .voice .crosshair".split()
21
+ )
22
+ PAYLOAD_DIRS = frozenset({"assets", "assets-src", "compiled-assets", "compiled-shaders",
23
+ ".cache", ".work", "target", "captures", "references", "packages"})
24
+ POINTER = re.compile(rb"version https://git-lfs.github.com/spec/v1\noid sha256:[0-9a-f]{64}\nsize [0-9]+\n")
25
+
26
+
27
+ def git(*args, input=None):
28
+ return subprocess.check_output(["git", *args], input=input)
29
+
30
+
31
+ def check_code(path, data):
32
+ name = PurePosixPath(path)
33
+ if is_asset(path) or any(part.lower() in PAYLOAD_DIRS for part in name.parts):
34
+ raise ValueError(f"{path}: asset/build/cache directory is not allowed in code Git")
35
+ if name.suffix.lower() in PAYLOAD_SUFFIXES or ".so." in name.name.lower():
36
+ raise ValueError(f"{path}: asset or binary extension belongs under assets/")
37
+ if len(data) > MAX_CODE_BYTES:
38
+ raise ValueError(f"{path}: exceeds the {MAX_CODE_BYTES}-byte code limit")
39
+ if b"\0" in data:
40
+ raise ValueError(f"{path}: binary data belongs under assets/")
41
+ try:
42
+ data.decode("utf-8")
43
+ except UnicodeDecodeError as exc:
44
+ raise ValueError(f"{path}: code/documentation must be UTF-8 text") from exc
45
+ if data.startswith(b"version https://git-lfs.github.com/spec/v1\n"):
46
+ raise ValueError(f"{path}: LFS pointers belong under assets/")
47
+
48
+
49
+ def check_asset(path, data):
50
+ # Git LFS specifies that an empty file is its own empty pointer.
51
+ # This contains no payload bytes and must still have filter=lfs attributes.
52
+ if data and not POINTER.fullmatch(data):
53
+ raise ValueError(f"{path}: asset Git object is not a canonical LFS pointer; run setup and git add --renormalize")
54
+
55
+
56
+ def is_asset(path):
57
+ name = PurePosixPath(path)
58
+ parts = name.parts
59
+ if parts[0] == "testdata":
60
+ return name.suffix.lower() != ".md" and name.name != ".gitignore"
61
+ if "fixtures" in parts[:-1] and "tests" in parts[:-1]:
62
+ return True
63
+ if parts[0] == "crates" and "data" in parts[:-1]:
64
+ return True
65
+ return (any(part in {"assets", "references", "assets-src", "compiled-assets", "compiled-shaders"}
66
+ for part in parts[:-1]) or name.suffix.lower() in PAYLOAD_SUFFIXES
67
+ or ".so." in name.name.lower())
68
+
69
+
70
+ @contextmanager
71
+ def blob_reader():
72
+ # One bounded streaming process, rather than two Git launches per asset.
73
+ process = subprocess.Popen(["git", "cat-file", "--batch"], stdin=subprocess.PIPE,
74
+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
75
+ try:
76
+ yield process
77
+ finally:
78
+ process.stdin.close()
79
+ process.stdout.close()
80
+ if process.poll() is None:
81
+ process.terminate()
82
+ process.wait()
83
+
84
+
85
+ def validate_blob(path, oid, reader):
86
+ reader.stdin.write(oid.encode() + b"\n")
87
+ reader.stdin.flush()
88
+ header = reader.stdout.readline().split()
89
+ if len(header) != 3 or header[1] != b"blob":
90
+ raise ValueError(f"{path}: invalid or missing Git blob")
91
+ size = int(header[2])
92
+ limit = 256 if is_asset(path) else MAX_CODE_BYTES
93
+ if size > limit:
94
+ raise ValueError(f"{path}: Git blob is {size} bytes; limit is {limit}")
95
+ data = reader.stdout.read(size)
96
+ if len(data) != size or reader.stdout.read(1) != b"\n":
97
+ raise ValueError(f"{path}: truncated Git batch output")
98
+ (check_asset if is_asset(path) else check_code)(path, data)
99
+
100
+
101
+ def check_index():
102
+ paths = []
103
+ with blob_reader() as reader:
104
+ for entry in git("ls-files", "--stage", "-z").split(b"\0"):
105
+ if not entry:
106
+ continue
107
+ header, path_bytes = entry.split(b"\t", 1)
108
+ permissions, oid, stage = header.decode().split()
109
+ path = os.fsdecode(path_bytes)
110
+ if stage != "0" or permissions not in {"100644", "100755"}:
111
+ raise ValueError(f"{path}: unresolved, symlink or nested repository entry is not allowed")
112
+ validate_blob(path, oid, reader)
113
+ paths.append(path)
114
+ if paths:
115
+ raw = git("check-attr", "--cached", "-z", "--stdin", "filter",
116
+ input=b"\0".join(os.fsencode(p) for p in paths) + b"\0").split(b"\0")
117
+ for i in range(0, len(raw) - 1, 3):
118
+ path, _, value = raw[i:i + 3]
119
+ path = os.fsdecode(path)
120
+ expected = b"lfs" if is_asset(path) else b"openwa-code"
121
+ if value != expected:
122
+ raise ValueError(f"{path}: expected filter={expected.decode()}, found {value.decode()}")
123
+ asset_paths = [p for p in paths if is_asset(p)]
124
+ if asset_paths:
125
+ raw = git("check-attr", "--cached", "-z", "--stdin", "text", "diff", "merge",
126
+ input=b"\0".join(os.fsencode(p) for p in asset_paths) + b"\0").split(b"\0")
127
+ for i in range(0, len(raw) - 1, 3):
128
+ path, attr, value = raw[i:i + 3]
129
+ expected = b"unset" if attr == b"text" else b"lfs"
130
+ if value != expected:
131
+ raise ValueError(f"{os.fsdecode(path)}: incorrect asset {attr.decode()} attribute")
132
+
133
+
134
+ def check_push_with_reader(reader):
135
+ # Validate all newly reachable commits, including files deleted later.
136
+ # A new remote branch checks its entire reachable history.
137
+ seen = set()
138
+ for line in sys.stdin:
139
+ _, local_oid, _, remote_oid = line.split()
140
+ if set(local_oid) == {"0"}:
141
+ continue
142
+ revisions = [local_oid]
143
+ if set(remote_oid) != {"0"}:
144
+ exists = subprocess.run(["git", "cat-file", "-e", remote_oid],
145
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
146
+ if exists.returncode == 0:
147
+ revisions.append("^" + remote_oid)
148
+ for commit in git("rev-list", *revisions).decode().splitlines():
149
+ for entry in git("ls-tree", "-rz", commit).split(b"\0"):
150
+ if not entry:
151
+ continue
152
+ header, path = entry.split(b"\t", 1)
153
+ permissions, kind, oid = header.decode().split()
154
+ if permissions not in {"100644", "100755"} or kind != "blob":
155
+ raise ValueError(f"{os.fsdecode(path)}: unsupported Git tree entry")
156
+ key = (os.fsdecode(path), oid)
157
+ if key not in seen:
158
+ validate_blob(*key, reader)
159
+ seen.add(key)
160
+
161
+
162
+ def check_push():
163
+ with blob_reader() as reader:
164
+ check_push_with_reader(reader)
165
+
166
+
167
+ def check_setup():
168
+ if not shutil.which("git-xet"):
169
+ raise ValueError("git-xet is missing; enter nix develop")
170
+ required = {
171
+ "filter.openwa-code.required": "true",
172
+ "filter.lfs.required": "true",
173
+ "lfs.customtransfer.xet.path": "git-xet",
174
+ "lfs.customtransfer.xet.args": "transfer",
175
+ "lfs.basictransfersonly": "false",
176
+ "core.hooksPath": ".githooks",
177
+ }
178
+ for key, value in required.items():
179
+ if git("config", "--local", "--get", key).decode().strip() != value:
180
+ raise ValueError(f"{key}: rerun scripts/setup-repository.py")
181
+ for key in ("filter.openwa-code.clean", "filter.lfs.clean", "filter.lfs.process"):
182
+ if not git("config", "--local", "--get", key).strip():
183
+ raise ValueError(f"{key}: missing filter driver")
184
+
185
+
186
+ def check_tree(directory):
187
+ """Check the asset-free source used by Nix validation/build derivations."""
188
+ if not directory:
189
+ raise ValueError("check-tree requires a source directory")
190
+ root = Path(directory)
191
+ for path in root.rglob("*"):
192
+ if path.is_symlink():
193
+ raise ValueError(f"{path}: source symlink is not allowed")
194
+ if path.is_file():
195
+ with path.open("rb") as stream:
196
+ check_code(path.relative_to(root).as_posix(), stream.read(MAX_CODE_BYTES + 1))
197
+ if "* text=auto eol=lf filter=openwa-code" not in (root / ".gitattributes").read_text().splitlines():
198
+ raise ValueError("Code export lacks the required default clean filter")
199
+
200
+
201
+ def main():
202
+ parser = argparse.ArgumentParser()
203
+ parser.add_argument("command", choices=["clean", "check", "push", "check-tree"])
204
+ parser.add_argument("--require-setup", action="store_true")
205
+ parser.add_argument("path", nargs="?")
206
+ args = parser.parse_args()
207
+ try:
208
+ if args.require_setup:
209
+ check_setup()
210
+ if args.command == "clean":
211
+ data = sys.stdin.buffer.read(MAX_CODE_BYTES + 1)
212
+ check_code(args.path or "<stdin>", data)
213
+ sys.stdout.buffer.write(data)
214
+ elif args.command == "check-tree":
215
+ check_tree(args.path)
216
+ elif args.command == "push":
217
+ check_push()
218
+ else:
219
+ check_index()
220
+ except (ValueError, subprocess.CalledProcessError) as exc:
221
+ print(f"Storage policy: {exc}", file=sys.stderr)
222
+ return 1
223
+ return 0
224
+
225
+
226
+ if __name__ == "__main__":
227
+ sys.exit(main())