Move README template in private and pass it
Browse files
job.py
CHANGED
|
@@ -126,40 +126,13 @@ def generate_samples(samples_dir: Path):
|
|
| 126 |
output.images[0].save(f'{samples_dir}/edited.png')
|
| 127 |
|
| 128 |
|
| 129 |
-
README_TEMPLATE = textwrap.dedent(
|
| 130 |
-
"""
|
| 131 |
-
## How to use
|
| 132 |
-
``` python
|
| 133 |
-
{MODEL_INIT}\n
|
| 134 |
-
{AOTI_LOAD}
|
| 135 |
-
```
|
| 136 |
-
|
| 137 |
-
## How to reproduce or customize
|
| 138 |
-
``` bash
|
| 139 |
-
pip install huggingface_hub
|
| 140 |
-
|
| 141 |
-
# Get the job file and edit if needed
|
| 142 |
-
hf download {REPO_ID} job.py
|
| 143 |
-
|
| 144 |
-
# Run the job
|
| 145 |
-
hf jobs uv run job.py \
|
| 146 |
-
--hardware {JOB_HARDWARE} # Change if needed \
|
| 147 |
-
--image {JOB_IMAGE} # ... \
|
| 148 |
-
-e PARENT_REPO_ID={REPO_ID} \
|
| 149 |
-
-e OUTPUT_REPO_ID=<output-repo-id> \
|
| 150 |
-
-e HF_TOKEN=<hf-token>
|
| 151 |
-
```
|
| 152 |
-
"""
|
| 153 |
-
)
|
| 154 |
-
|
| 155 |
-
|
| 156 |
# ========================================================
|
| 157 |
# ===== Meta Job below this line. Do not edit unless =====
|
| 158 |
# ========================================================
|
| 159 |
|
| 160 |
|
| 161 |
def capture_model_init_region():
|
| 162 |
-
source = inspect.getsource(sys.modules[
|
| 163 |
return (source
|
| 164 |
.split('# README::MODEL_INIT::START')[1]
|
| 165 |
.split('# README::MODEL_INIT::END')[0]
|
|
@@ -198,6 +171,7 @@ def create_aoti_repo(
|
|
| 198 |
module_expr: str,
|
| 199 |
compile_module: Callable[[torch.nn.Module, Path]],
|
| 200 |
generate_samples: Callable[[Path]],
|
|
|
|
| 201 |
aoti_loader: Callable[[torch.nn.Module, Path], None] | None = None,
|
| 202 |
):
|
| 203 |
"""
|
|
@@ -232,7 +206,7 @@ def create_aoti_repo(
|
|
| 232 |
repo_id=output_repo_id,
|
| 233 |
aoti_loader=aoti_loader,
|
| 234 |
)
|
| 235 |
-
readme_contents =
|
| 236 |
MODEL_INIT=capture_model_init_region(),
|
| 237 |
AOTI_LOAD=aoti_load_readme,
|
| 238 |
REPO_ID=output_repo_id,
|
|
@@ -266,6 +240,39 @@ def create_aoti_repo(
|
|
| 266 |
upload_folder(repo_id=output_repo_id, folder_path=tempdir)
|
| 267 |
|
| 268 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
if __name__ == '__main__':
|
| 270 |
create_aoti_repo(
|
| 271 |
output_repo_id=os.environ['OUTPUT_REPO_ID'],
|
|
@@ -273,5 +280,6 @@ if __name__ == '__main__':
|
|
| 273 |
module_expr='pipeline.transformer',
|
| 274 |
compile_module=compile_module,
|
| 275 |
generate_samples=generate_samples,
|
|
|
|
| 276 |
aoti_loader=aoti_loader,
|
| 277 |
)
|
|
|
|
| 126 |
output.images[0].save(f'{samples_dir}/edited.png')
|
| 127 |
|
| 128 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
# ========================================================
|
| 130 |
# ===== Meta Job below this line. Do not edit unless =====
|
| 131 |
# ========================================================
|
| 132 |
|
| 133 |
|
| 134 |
def capture_model_init_region():
|
| 135 |
+
source = inspect.getsource(sys.modules['__main__'])
|
| 136 |
return (source
|
| 137 |
.split('# README::MODEL_INIT::START')[1]
|
| 138 |
.split('# README::MODEL_INIT::END')[0]
|
|
|
|
| 171 |
module_expr: str,
|
| 172 |
compile_module: Callable[[torch.nn.Module, Path]],
|
| 173 |
generate_samples: Callable[[Path]],
|
| 174 |
+
readme_template: str,
|
| 175 |
aoti_loader: Callable[[torch.nn.Module, Path], None] | None = None,
|
| 176 |
):
|
| 177 |
"""
|
|
|
|
| 206 |
repo_id=output_repo_id,
|
| 207 |
aoti_loader=aoti_loader,
|
| 208 |
)
|
| 209 |
+
readme_contents = readme_template.format(
|
| 210 |
MODEL_INIT=capture_model_init_region(),
|
| 211 |
AOTI_LOAD=aoti_load_readme,
|
| 212 |
REPO_ID=output_repo_id,
|
|
|
|
| 240 |
upload_folder(repo_id=output_repo_id, folder_path=tempdir)
|
| 241 |
|
| 242 |
|
| 243 |
+
README_TEMPLATE = textwrap.dedent(
|
| 244 |
+
"""
|
| 245 |
+
## How to use
|
| 246 |
+
``` python
|
| 247 |
+
{MODEL_INIT}\n
|
| 248 |
+
{AOTI_LOAD}
|
| 249 |
+
```
|
| 250 |
+
|
| 251 |
+
## How to reproduce or customize
|
| 252 |
+
``` bash
|
| 253 |
+
pip install huggingface_hub
|
| 254 |
+
|
| 255 |
+
# Get the job file and edit if needed
|
| 256 |
+
hf download {REPO_ID} job.py
|
| 257 |
+
|
| 258 |
+
# Run the job
|
| 259 |
+
hf jobs uv run job.py \
|
| 260 |
+
--hardware {JOB_HARDWARE} # Change if needed \
|
| 261 |
+
--image {JOB_IMAGE} # ... \
|
| 262 |
+
-e PARENT_REPO_ID={REPO_ID} \
|
| 263 |
+
-e OUTPUT_REPO_ID=<output-repo-id> \
|
| 264 |
+
-e HF_TOKEN=<hf-token>
|
| 265 |
+
```
|
| 266 |
+
|
| 267 |
+
## Environment
|
| 268 |
+
<details>
|
| 269 |
+
<summary>Click to expand</summary>
|
| 270 |
+
{ENVIRONMENT}
|
| 271 |
+
</details>
|
| 272 |
+
"""
|
| 273 |
+
)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
if __name__ == '__main__':
|
| 277 |
create_aoti_repo(
|
| 278 |
output_repo_id=os.environ['OUTPUT_REPO_ID'],
|
|
|
|
| 280 |
module_expr='pipeline.transformer',
|
| 281 |
compile_module=compile_module,
|
| 282 |
generate_samples=generate_samples,
|
| 283 |
+
readme_template=README_TEMPLATE, # TODO: allow for extra template data dict?
|
| 284 |
aoti_loader=aoti_loader,
|
| 285 |
)
|