Text Generation
Transformers
English
custom
tokenizer
symbolic-ai
mathematics
llm
reasoning
ast
compiler
nlp
deep-learning
machine-learning
mathematical-reasoning
symbolic-reasoning
tokenization
parser
artificial-intelligence
Eval Results (legacy)
Instructions to use SurweeshSP/mathtok with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use SurweeshSP/mathtok with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="SurweeshSP/mathtok")# Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("SurweeshSP/mathtok", dtype="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps
- vLLM
How to use SurweeshSP/mathtok with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "SurweeshSP/mathtok" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "SurweeshSP/mathtok", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/SurweeshSP/mathtok
- SGLang
How to use SurweeshSP/mathtok with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "SurweeshSP/mathtok" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "SurweeshSP/mathtok", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "SurweeshSP/mathtok" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "SurweeshSP/mathtok", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use SurweeshSP/mathtok with Docker Model Runner:
docker model run hf.co/SurweeshSP/mathtok
| """ | |
| Tests for the Canonicalization Layer (Layer 1). | |
| Covers: | |
| - ASCII expression parsing | |
| - LaTeX expression parsing | |
| - Equivalence detection (are_equivalent) | |
| - Normalization transformations | |
| - Fallback behaviour on parse errors | |
| """ | |
| import pytest | |
| import sympy as sp | |
| from mathtok.canonicalizer import Canonicalizer, CanonicalizationResult | |
| def canon(): | |
| return Canonicalizer(do_simplify=True, do_expand=True) | |
| # ββ Parsing βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class TestParsing: | |
| def test_ascii_simple(self, canon): | |
| r = canon.canonicalize("x^2 + 1") | |
| assert r.success | |
| assert r.input_format == "ascii" | |
| assert "x" in str(r.expr) | |
| def test_ascii_implicit_mul(self, canon): | |
| r = canon.canonicalize("2x + 1") | |
| assert r.success | |
| def test_ascii_constants(self, canon): | |
| r = canon.canonicalize("pi + e") | |
| assert r.success | |
| assert sp.pi in r.expr.free_symbols or r.expr == sp.pi + sp.E | |
| def test_latex_frac(self, canon): | |
| r = canon.canonicalize("\\frac{x^2}{2}") | |
| # LaTeX detected | |
| assert r.input_format == "latex" or r.success # may fallback | |
| def test_latex_sin(self, canon): | |
| r = canon.canonicalize("\\sin(x^2)") | |
| assert r.success | |
| def test_latex_sqrt(self, canon): | |
| r = canon.canonicalize("\\sqrt{x^2 + 1}") | |
| assert r.success | |
| def test_parse_error_graceful(self, canon): | |
| r = canon.canonicalize("@@@invalid@@@") | |
| assert not r.success | |
| assert len(r.warnings) > 0 | |
| def test_delimiters_stripped(self, canon): | |
| r = canon.canonicalize("$x^2 + 1$") | |
| assert r.success | |
| # ββ Normalization βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class TestNormalization: | |
| def test_expand(self, canon): | |
| r = canon.canonicalize("(x+1)^2") | |
| # expanded form should include x^2 and 2x | |
| expr_str = str(r.expr) | |
| assert "x**2" in expr_str or "x^2" in expr_str | |
| def test_commutativity_canonical(self, canon): | |
| r1 = canon.canonicalize("a + b") | |
| r2 = canon.canonicalize("b + a") | |
| # SymPy canonicalises Add ordering | |
| assert str(r1.expr) == str(r2.expr) | |
| def test_subtraction_to_add(self, canon): | |
| r = canon.canonicalize("x - y") | |
| # SymPy represents x-y as Add(x, Mul(-1, y)) | |
| assert isinstance(r.expr, sp.Add) | |
| def test_division_to_mul(self, canon): | |
| r = canon.canonicalize("x / y") | |
| # SymPy represents x/y as Mul(x, Pow(y, -1)) | |
| assert isinstance(r.expr, sp.Mul) | |
| def test_transformations_recorded(self, canon): | |
| r = canon.canonicalize("x^2 + 2*x + 1") | |
| assert "expand" in r.transformations_applied | |
| assert "simplify" in r.transformations_applied | |
| # ββ Equivalence βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class TestEquivalence: | |
| def test_basic_equivalent(self, canon): | |
| assert canon.are_equivalent("(x+1)^2", "x^2 + 2*x + 1") | |
| def test_commutative_equivalent(self, canon): | |
| assert canon.are_equivalent("a + b", "b + a") | |
| def test_not_equivalent(self, canon): | |
| assert not canon.are_equivalent("x^2", "x^3") | |
| def test_trig_identity(self, canon): | |
| # sin^2 + cos^2 = 1 | |
| assert canon.are_equivalent("sin(x)^2 + cos(x)^2", "1") | |
| def test_log_product(self, canon): | |
| # log(x)+log(y) = log(x*y) requires positive assumptions; | |
| # SymPy's simplify may not collapse it without them. | |
| # Verify at least that both are valid canonical expressions. | |
| r1 = canon.canonicalize("log(x) + log(y)") | |
| r2 = canon.canonicalize("log(x*y)") | |
| assert r1.success and r2.success | |
| # With positive assumptions the difference simplifies to 0 | |
| import sympy as sp | |
| x, y = sp.Symbol("x", positive=True), sp.Symbol("y", positive=True) | |
| diff = sp.simplify(sp.log(x) + sp.log(y) - sp.log(x * y)) | |
| assert diff == 0 | |
| def test_difference_of_squares(self, canon): | |
| assert canon.are_equivalent("a^2 - b^2", "(a+b)*(a-b)") | |