Hugging Face's logo

Spaces:
slamos
/
bc-test
Paused

App Files Community
bc-test
File size: 4,495 Bytes
7b4cc2e
 
 
 
 
 
8159f14
 
 
 
 
 
 
 
 
 
7b4cc2e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
install:
	pip install -r requirements.txt

install-runpod:
	pip install --ignore-installed blinker && pip install -r requirements.txt

DATA_GDRIVE_ID ?= 1e0ld3FXNYJUQFre0fwgQOUTnTaO_ONnJ

download_data:
	gdown $(DATA_GDRIVE_ID) -O data.zip
	unzip -o data.zip
	rm data.zip

zip_data:
	zip -r data.zip data/

download_tedseg:
	python -m src.datasets.download_tedseg

download_pubmed:
	python -m src.datasets.download_pubmed --k 100

download_wikipedia:
	python -m src.datasets.download_wikipedia --k 100

preprocess_all: preprocess_pubmed preprocess_wikipedia preprocess_gutenberg preprocess_ted

preprocess_pubmed:
	python -m src.datasets.preprocess pubmed --input data/pubmed/raw/pubmed_raw.jsonl --output data/pubmed/pubmed_data.jsonl

preprocess_wikipedia:
	python -m src.datasets.preprocess wikipedia --input data/wikipedia/raw/wikipedia_raw.jsonl --output data/wikipedia/wikipedia_data.jsonl

preprocess_gutenberg:
	python -m src.datasets.preprocess gutenberg --raw_dir data/gutenberg/raw --output data/gutenberg/gutenberg_data.jsonl

preprocess_ted:
	python -m src.datasets.preprocess ted --input_dir data/tedseg/raw --output_dir data/tedseg

create_recipes:
	python -m src.datasets.create_recipes_dataset

sentence_split_all: sentence_split_gutenberg sentence_split_pubmed sentence_split_wikipedia sentence_split_recipes

sentence_split_gutenberg:
	python -m src.datasets.sentence_splitter gutenberg --input data/gutenberg/gutenberg_data.jsonl --output data/gutenberg/gutenberg_sentences.jsonl --device cuda

sentence_split_ted:
	python -m src.datasets.sentence_splitter ted --input_dir data/tedseg/raw --output_dir data/tedseg --device cuda

sentence_split_pubmed:
	python -m src.datasets.sentence_splitter pubmed --input data/pubmed/pubmed_data.jsonl --output data/pubmed/pubmed_sentences.jsonl --device cuda

sentence_split_wikipedia:
	python -m src.datasets.sentence_splitter wikipedia --input data/wikipedia/wikipedia_data.jsonl --output data/wikipedia/wikipedia_sentences.jsonl --device cuda

sentence_split_recipes:
	python -m src.datasets.sentence_splitter recipes --input data/recipes/recipes_data.jsonl --output data/recipes/recipes_sentences.jsonl --device cuda

build_recipes_pairs:
	python -m src.datasets.build_pairs recipes

train_distilbert:
	python -m src.models.train --model distilbert --out checkpoints/distilbert

train_bert:
	python -m src.models.train --model bert --out checkpoints/bert

train_deberta:
	python -m src.models.train --model deberta --out checkpoints/deberta

train_all: train_distilbert train_bert train_deberta

export_distilbert:
	python -m src.models.export_and_download --checkpoint checkpoints/distilbert/best

export_bert:
	python -m src.models.export_and_download --checkpoint checkpoints/bert/best

export_deberta:
	python -m src.models.export_and_download --checkpoint checkpoints/deberta/best

export_all: export_distilbert export_bert export_deberta

upload_distilbert:
	python -m src.models.export_and_download --checkpoint checkpoints/distilbert/best --repo slamos/bc-models-distilbert

upload_bert:
	python -m src.models.export_and_download --checkpoint checkpoints/bert/best --repo slamos/bc-models-bert

upload_deberta:
	python -m src.models.export_and_download --checkpoint checkpoints/deberta/best --repo slamos/bc-models-deberta

upload_all: upload_distilbert upload_bert upload_deberta

download_distilbert:
	python -m src.models.export_and_download --download distilbert

download_bert:
	python -m src.models.export_and_download --download bert

download_deberta:
	python -m src.models.export_and_download --download deberta

download_all: download_distilbert download_bert download_deberta

inference_distilbert:
	python -m src.models.inference --model distilbert

inference_bert:
	python -m src.models.inference --model bert

inference_deberta:
	python -m src.models.inference --model deberta

inference_local_distilbert:
	python -m src.models.inference --model distilbert --local

inference_local_bert:
	python -m src.models.inference --model bert --local

inference_local_deberta:
	python -m src.models.inference --model deberta --local

test-api:
	python -m pytest tests/test_api.py -v

test-dataset:
	python -m pytest tests/test_dataset.py -v

test-pipelines:
	python -m pytest tests/test_pipelines.py -v

test-all:
	python -m pytest tests/ -v

clean:
	rm -rf checkpoints/distilbert checkpoints/bert checkpoints/deberta checkpoints/plots

run-be:
	uvicorn main:app --host 0.0.0.0 --port 8000

run-fe:
	streamlit run app.py