Add GGUF exports for analysis (F16, Q8_0, Q5_K_M, Q4_K_M)
#7
by pylotlight - opened
.gitattributes
CHANGED
|
@@ -46,7 +46,3 @@ assets/plays_snake.gif filter=lfs diff=lfs merge=lfs -text
|
|
| 46 |
assets/training_distillation.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
assets/progress.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
assets/training_thinking.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
-
assets/community/chart_speed_accuracy_size.png filter=lfs diff=lfs merge=lfs -text
|
| 50 |
-
assets/community/chart_resolution_adaptive.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
-
assets/community/chart_stage_timing.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
-
assets/community/chart_inductor_cache.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 46 |
assets/training_distillation.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
assets/progress.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
assets/training_thinking.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
assets/community/chart_inductor_cache.png
DELETED
Git LFS Details
|
assets/community/chart_resolution_adaptive.png
DELETED
Git LFS Details
|
assets/community/chart_speed_accuracy_size.png
DELETED
Git LFS Details
|
assets/community/chart_stage_timing.png
DELETED
Git LFS Details
|
thx02/model.py
CHANGED
|
@@ -32,7 +32,6 @@ from .types import Request, Response
|
|
| 32 |
|
| 33 |
if TYPE_CHECKING:
|
| 34 |
from .fast import FastPath
|
| 35 |
-
from .mps_fast import MPSFastPath
|
| 36 |
|
| 37 |
#: Default per-sequence token limit. Longer requests are split into question chunks (each with the full state).
|
| 38 |
DEFAULT_MAX_LENGTH = 262_144 # backbone limit (max_position_embeddings); training used sequences up to about 32k tokens
|
|
@@ -113,7 +112,7 @@ class THXDecisionModel:
|
|
| 113 |
checkpoint: Path,
|
| 114 |
*,
|
| 115 |
device: str = "cuda",
|
| 116 |
-
fast: bool | FastOptions
|
| 117 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 118 |
max_pixels: int | None = None,
|
| 119 |
batch_tokens: int = 32768,
|
|
@@ -161,17 +160,11 @@ class THXDecisionModel:
|
|
| 161 |
self.backbone.requires_grad_(False).eval().to(self.device)
|
| 162 |
self.readout.requires_grad_(False).eval().to(self.device)
|
| 163 |
|
| 164 |
-
self.fast: FastPath |
|
| 165 |
if fast:
|
| 166 |
-
|
| 167 |
-
# No CUDA: bucketed static shapes + a compiled language model (no FLA, no CUDA graphs).
|
| 168 |
-
from .mps_fast import MPSFastOptions, MPSFastPath
|
| 169 |
|
| 170 |
-
|
| 171 |
-
else:
|
| 172 |
-
from .fast import FastPath
|
| 173 |
-
|
| 174 |
-
self.fast = FastPath(self, fast if isinstance(fast, FastOptions) else FastOptions())
|
| 175 |
|
| 176 |
@classmethod
|
| 177 |
def from_pretrained(
|
|
@@ -179,7 +172,7 @@ class THXDecisionModel:
|
|
| 179 |
path_or_repo: str | Path,
|
| 180 |
*,
|
| 181 |
device: str = "cuda",
|
| 182 |
-
fast: bool | FastOptions
|
| 183 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 184 |
max_pixels: int | None = None,
|
| 185 |
revision: str | None = None,
|
|
@@ -189,8 +182,7 @@ class THXDecisionModel:
|
|
| 189 |
"""Load a local checkpoint directory or a Hugging Face repository.
|
| 190 |
|
| 191 |
fast: ``False`` for the reference path (padded batches, exact evaluated computation); ``True`` or a
|
| 192 |
-
``FastOptions`` for the packed
|
| 193 |
-
MPS path (same results up to bf16 noise, several times faster).
|
| 194 |
max_length: per-sequence token limit; longer requests are split into question chunks, see the README.
|
| 195 |
max_pixels: image pixel cap (default: the checkpoint's ``max_pixels``, else 262,144).
|
| 196 |
"""
|
|
|
|
| 32 |
|
| 33 |
if TYPE_CHECKING:
|
| 34 |
from .fast import FastPath
|
|
|
|
| 35 |
|
| 36 |
#: Default per-sequence token limit. Longer requests are split into question chunks (each with the full state).
|
| 37 |
DEFAULT_MAX_LENGTH = 262_144 # backbone limit (max_position_embeddings); training used sequences up to about 32k tokens
|
|
|
|
| 112 |
checkpoint: Path,
|
| 113 |
*,
|
| 114 |
device: str = "cuda",
|
| 115 |
+
fast: bool | FastOptions = False,
|
| 116 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 117 |
max_pixels: int | None = None,
|
| 118 |
batch_tokens: int = 32768,
|
|
|
|
| 160 |
self.backbone.requires_grad_(False).eval().to(self.device)
|
| 161 |
self.readout.requires_grad_(False).eval().to(self.device)
|
| 162 |
|
| 163 |
+
self.fast: FastPath | None = None
|
| 164 |
if fast:
|
| 165 |
+
from .fast import FastPath
|
|
|
|
|
|
|
| 166 |
|
| 167 |
+
self.fast = FastPath(self, fast if isinstance(fast, FastOptions) else FastOptions())
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
@classmethod
|
| 170 |
def from_pretrained(
|
|
|
|
| 172 |
path_or_repo: str | Path,
|
| 173 |
*,
|
| 174 |
device: str = "cuda",
|
| 175 |
+
fast: bool | FastOptions = False,
|
| 176 |
max_length: int | None = DEFAULT_MAX_LENGTH,
|
| 177 |
max_pixels: int | None = None,
|
| 178 |
revision: str | None = None,
|
|
|
|
| 182 |
"""Load a local checkpoint directory or a Hugging Face repository.
|
| 183 |
|
| 184 |
fast: ``False`` for the reference path (padded batches, exact evaluated computation); ``True`` or a
|
| 185 |
+
``FastOptions`` for the packed path (same results up to bf16 noise, several times faster).
|
|
|
|
| 186 |
max_length: per-sequence token limit; longer requests are split into question chunks, see the README.
|
| 187 |
max_pixels: image pixel cap (default: the checkpoint's ``max_pixels``, else 262,144).
|
| 188 |
"""
|
thx02/mps_fast.py
DELETED
|
@@ -1,217 +0,0 @@
|
|
| 1 |
-
"""MPS fast path: bucketed static shapes with a compiled language model.
|
| 2 |
-
|
| 3 |
-
The packaged fast path (:mod:`thx02.fast`) needs CUDA: varlen FlashAttention, ``fla``'s chunked
|
| 4 |
-
gated delta rule and CUDA graphs. On Apple Silicon only the reference path runs, and its gated-delta
|
| 5 |
-
linear-attention layers fall back to the sequential PyTorch implementation, which is the main
|
| 6 |
-
bottleneck.
|
| 7 |
-
|
| 8 |
-
This path takes the same shape as the idea: decision requests are a single forward pass with no
|
| 9 |
-
generation loop, so a few length buckets plus ``torch.compile`` give Inductor stable graphs on MPS.
|
| 10 |
-
|
| 11 |
-
What is compiled, and what is not
|
| 12 |
-
Only ``backbone.language_model`` is compiled. Compiling the full backbone hits an Inductor Metal
|
| 13 |
-
codegen bug on image inputs (``no 'buffer' resource location available for 'error_buf'``), and the
|
| 14 |
-
vision tower is the cheap, cacheable half anyway. Embedding lookup, image-feature scatter and the
|
| 15 |
-
M-RoPE positions run eagerly at the request's natural length; the LM then runs over a fixed
|
| 16 |
-
``(1, bucket)`` shape with right padding (masked, never read out).
|
| 17 |
-
|
| 18 |
-
Right padding is safe here because the checkpoint's full-attention layers are noncausal with a
|
| 19 |
-
padding mask and its linear-attention layers are causal: pad tokens to the right cannot influence
|
| 20 |
-
real tokens in either layer type.
|
| 21 |
-
|
| 22 |
-
Batching
|
| 23 |
-
One chunk per forward pass (batch = 1). ``torch.compile(dynamic=False)`` specializes per shape, so
|
| 24 |
-
a compiled batch dimension would need one graph per (batch, bucket) pair. MPS is a single device;
|
| 25 |
-
the speedup comes from compile, not from batching. Requests are still answered in input order and
|
| 26 |
-
the token usage reported is the natural length, not the bucket.
|
| 27 |
-
|
| 28 |
-
Over-length chunks
|
| 29 |
-
A chunk longer than the largest bucket runs eagerly at its natural length (no new graph, no
|
| 30 |
-
recompile). Raise ``buckets`` to cover longer inputs, or accept the eager fallback for the rare
|
| 31 |
-
long request.
|
| 32 |
-
"""
|
| 33 |
-
|
| 34 |
-
from __future__ import annotations
|
| 35 |
-
|
| 36 |
-
import time
|
| 37 |
-
from dataclasses import dataclass, field
|
| 38 |
-
from typing import Any, Iterator
|
| 39 |
-
|
| 40 |
-
import torch
|
| 41 |
-
|
| 42 |
-
from .protocol import Encoded, Request
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
@dataclass(frozen=True)
|
| 46 |
-
class MPSFastOptions:
|
| 47 |
-
"""Settings of the MPS fast path.
|
| 48 |
-
|
| 49 |
-
buckets: static sequence lengths to compile for. A chunk is right-padded up to the smallest bucket
|
| 50 |
-
that fits; longer chunks run eagerly. The default steps by ~1.25x from 128 to 1024, which
|
| 51 |
-
covers short questions as well as one 768 px image (~719 tokens) with at most 25% padding.
|
| 52 |
-
Power-of-two buckets are a trap: padding a 606-token request to 768 costs more than
|
| 53 |
-
compile saves, and the fast path ends up no faster than the reference path. The small
|
| 54 |
-
buckets matter just as much: a 90-token request padded to 256 is *slower* than the
|
| 55 |
-
reference path, while a 128 bucket makes it 1.36x faster.
|
| 56 |
-
compile: ``False`` keeps the same bucketed/eager-input layout without ``torch.compile``; useful as
|
| 57 |
-
an A/B baseline for the padding scheme alone.
|
| 58 |
-
dynamic: pass ``dynamic=True`` to ``torch.compile``. Slower to compile and historically fragile
|
| 59 |
-
with the gated-delta fallback on Metal; ``False`` (one graph per bucket) is the supported
|
| 60 |
-
default.
|
| 61 |
-
"""
|
| 62 |
-
|
| 63 |
-
buckets: tuple[int, ...] = (128, 192, 256, 320, 384, 448, 512, 640, 768, 1024)
|
| 64 |
-
compile: bool = True
|
| 65 |
-
dynamic: bool = False
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
@dataclass
|
| 69 |
-
class MPSFastStats:
|
| 70 |
-
"""Counters, for validation and logging (never used for control flow)."""
|
| 71 |
-
|
| 72 |
-
compile_s: float = 0.0
|
| 73 |
-
graphs: int = 0
|
| 74 |
-
bucketed: int = 0
|
| 75 |
-
eager: int = 0
|
| 76 |
-
per_bucket: dict[int, int] = field(default_factory=dict)
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
class MPSFastPath:
|
| 80 |
-
"""Bucketed, LM-compiled replacement for ``FastPath`` on MPS.
|
| 81 |
-
|
| 82 |
-
Implements the same contract as ``FastPath``: ``run(requests)`` yields
|
| 83 |
-
``(request, {question_key: logits}, chunks)`` and ``forward(chunks)`` returns one list of
|
| 84 |
-
per-question logit rows per chunk. ``warmup()`` compiles every bucket up front.
|
| 85 |
-
"""
|
| 86 |
-
|
| 87 |
-
def __init__(self, decider: Any, options: MPSFastOptions | None = None) -> None:
|
| 88 |
-
if decider.device.type != "mps":
|
| 89 |
-
raise ValueError("The MPS fast path needs a Metal (MPS) device")
|
| 90 |
-
self.decider = decider
|
| 91 |
-
self.options = options or MPSFastOptions()
|
| 92 |
-
self.backbone = decider.backbone
|
| 93 |
-
self.lm = self.backbone.language_model
|
| 94 |
-
self.buckets = tuple(sorted({int(b) for b in self.options.buckets if int(b) > 0}))
|
| 95 |
-
if not self.buckets:
|
| 96 |
-
raise ValueError("The MPS fast path needs at least one positive bucket")
|
| 97 |
-
self.stats = MPSFastStats()
|
| 98 |
-
self._compiled: Any = None
|
| 99 |
-
self._seen_buckets: set[int] = set()
|
| 100 |
-
if self.options.compile:
|
| 101 |
-
torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 64)
|
| 102 |
-
|
| 103 |
-
# ------------------------------------------------------------------ setup
|
| 104 |
-
|
| 105 |
-
def reset(self) -> None:
|
| 106 |
-
"""Drop compiled graphs (e.g. after weights were swapped for a quantized copy)."""
|
| 107 |
-
self._compiled = None
|
| 108 |
-
self._seen_buckets.clear()
|
| 109 |
-
self.stats.graphs = 0
|
| 110 |
-
self.stats.compile_s = 0.0
|
| 111 |
-
|
| 112 |
-
def _fn(self) -> Any:
|
| 113 |
-
if not self.options.compile:
|
| 114 |
-
return self.lm
|
| 115 |
-
if self._compiled is None:
|
| 116 |
-
self._compiled = torch.compile(self.lm, dynamic=self.options.dynamic)
|
| 117 |
-
return self._compiled
|
| 118 |
-
|
| 119 |
-
def warmup(self) -> None:
|
| 120 |
-
"""Compile and run every bucket once, so the first real request does not pay for it."""
|
| 121 |
-
encoder = self.decider.encoder
|
| 122 |
-
request = {"state": "warmup", "questions": {"q": {"type": "noul", "instructions": "Warm up?"}}}
|
| 123 |
-
chunk = encoder.plan(request, None)[0]
|
| 124 |
-
for bucket in self.buckets:
|
| 125 |
-
self._forward_padded(chunk, bucket)
|
| 126 |
-
torch.mps.synchronize()
|
| 127 |
-
|
| 128 |
-
# -------------------------------------------------------------- scheduling
|
| 129 |
-
|
| 130 |
-
def run(self, requests: list[Request]) -> Iterator[tuple[Request, dict[str, list[float]], list[Encoded]]]:
|
| 131 |
-
"""Yield (request, logits per question, chunks) in input order."""
|
| 132 |
-
encoder = self.decider.encoder
|
| 133 |
-
for request in requests:
|
| 134 |
-
encoder.validate(request)
|
| 135 |
-
plan = encoder.plan(request, self.decider.max_length)
|
| 136 |
-
results: dict[str, list[float]] = {}
|
| 137 |
-
for chunk, rows in zip(plan, self.forward(plan), strict=True):
|
| 138 |
-
results.update(zip(chunk.keys, rows, strict=True))
|
| 139 |
-
yield request, {key: results[key] for key in request["questions"]}, plan
|
| 140 |
-
|
| 141 |
-
# -------------------------------------------------------------- forward
|
| 142 |
-
|
| 143 |
-
def forward(self, chunks: list[Encoded]) -> list[list[list[float]]]:
|
| 144 |
-
return [self._forward_one(chunk) for chunk in chunks]
|
| 145 |
-
|
| 146 |
-
def _forward_one(self, chunk: Encoded) -> list[list[float]]:
|
| 147 |
-
bucket = next((size for size in self.buckets if size >= chunk.length), None)
|
| 148 |
-
if bucket is None: # over-length: eager at natural length, no new graph
|
| 149 |
-
self.stats.eager += 1
|
| 150 |
-
return self._run(chunk, None)
|
| 151 |
-
self.stats.bucketed += 1
|
| 152 |
-
self.stats.per_bucket[bucket] = self.stats.per_bucket.get(bucket, 0) + 1
|
| 153 |
-
return self._run(chunk, bucket)
|
| 154 |
-
|
| 155 |
-
def _forward_padded(self, chunk: Encoded, bucket: int) -> list[list[float]]:
|
| 156 |
-
return self._run(chunk, bucket)
|
| 157 |
-
|
| 158 |
-
@torch.inference_mode()
|
| 159 |
-
def _run(self, chunk: Encoded, bucket: int | None) -> list[list[float]]:
|
| 160 |
-
device = self.decider.device
|
| 161 |
-
embeds, positions, mask = self._inputs(chunk, bucket)
|
| 162 |
-
fn = self._fn() if bucket is not None else self.lm
|
| 163 |
-
first = bucket is not None and self.options.compile and bucket not in self._seen_buckets
|
| 164 |
-
started = time.perf_counter() if first else 0.0
|
| 165 |
-
hidden = fn(inputs_embeds=embeds, attention_mask=mask, position_ids=positions, use_cache=False)
|
| 166 |
-
if first: # the call above compiled and ran; the synchronize makes the timing honest
|
| 167 |
-
torch.mps.synchronize()
|
| 168 |
-
self._seen_buckets.add(bucket)
|
| 169 |
-
self.stats.graphs += 1
|
| 170 |
-
self.stats.compile_s += time.perf_counter() - started
|
| 171 |
-
markers = chunk.markers.to(device)
|
| 172 |
-
logits = self.decider.readout(hidden.last_hidden_state[0, markers]).float().cpu()
|
| 173 |
-
out, cursor = [], 0
|
| 174 |
-
for count in chunk.counts:
|
| 175 |
-
out.append(logits[cursor, :count].tolist())
|
| 176 |
-
cursor += 1
|
| 177 |
-
return out
|
| 178 |
-
|
| 179 |
-
def _inputs(self, chunk: Encoded, bucket: int | None) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 180 |
-
"""Eager embeddings + vision scatter + M-RoPE positions, then right-pad to the bucket."""
|
| 181 |
-
device = self.decider.device
|
| 182 |
-
ids = chunk.input_ids.to(device)[None]
|
| 183 |
-
embeds = self.backbone.get_input_embeddings()(ids)
|
| 184 |
-
extra: dict[str, Any] = {}
|
| 185 |
-
if chunk.pixel_values is not None:
|
| 186 |
-
grid = chunk.image_grid_thw.to(device)
|
| 187 |
-
features = self.backbone.get_image_features(
|
| 188 |
-
chunk.pixel_values.to(device), grid, return_dict=True
|
| 189 |
-
).pooler_output
|
| 190 |
-
features = torch.cat(features, dim=0).to(embeds.dtype)
|
| 191 |
-
image_mask, _ = self.backbone.get_placeholder_mask(ids, inputs_embeds=embeds, image_features=features)
|
| 192 |
-
embeds = embeds.masked_scatter(image_mask, features)
|
| 193 |
-
extra["image_grid_thw"] = grid
|
| 194 |
-
extra["mm_token_type_ids"] = chunk.mm_token_type_ids.to(device)[None]
|
| 195 |
-
mask = torch.ones((1, chunk.length), dtype=torch.long, device=device)
|
| 196 |
-
positions = self.backbone.compute_3d_position_ids(
|
| 197 |
-
input_ids=ids, inputs_embeds=embeds, attention_mask=mask, **extra
|
| 198 |
-
)
|
| 199 |
-
if positions is None: # text-only: the model's own 1-D positions, broadcast over the 3 M-RoPE axes
|
| 200 |
-
positions = torch.arange(chunk.length, device=device).view(1, 1, -1).expand(3, 1, -1)
|
| 201 |
-
if bucket is None or bucket == chunk.length:
|
| 202 |
-
return embeds, positions, mask
|
| 203 |
-
return self._pad(embeds, positions, mask, bucket)
|
| 204 |
-
|
| 205 |
-
@staticmethod
|
| 206 |
-
def _pad(
|
| 207 |
-
embeds: torch.Tensor, positions: torch.Tensor, mask: torch.Tensor, bucket: int
|
| 208 |
-
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 209 |
-
"""Right-pad to a static length. Padding is masked and never read out, so zeros are enough."""
|
| 210 |
-
length = embeds.shape[1]
|
| 211 |
-
e = torch.zeros((1, bucket, embeds.shape[-1]), dtype=embeds.dtype, device=embeds.device)
|
| 212 |
-
e[:, :length] = embeds
|
| 213 |
-
m = torch.zeros((1, bucket), dtype=mask.dtype, device=mask.device)
|
| 214 |
-
m[:, :length] = mask
|
| 215 |
-
p = torch.zeros((*positions.shape[:-1], bucket), dtype=positions.dtype, device=positions.device)
|
| 216 |
-
p[..., :length] = positions
|
| 217 |
-
return e, p, m
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|