Spaces:
Paused
Paused
| _A='/home/user' | |
| import os,subprocess | |
| from pathlib import Path | |
| from loguru import logger | |
| from services.utils import deobfuscate_secret | |
| METRICS_DIR='/home/user/.torch_metrics' | |
| CONFIG_PATH='/home/user/litellm.yaml'if Path(_A).exists()else'litellm.yaml' | |
| PORT=8080 | |
| PREFIX='[llm-proxy]' | |
| def _load_keys(): | |
| I='keys';N=[Path('llm_keys.yaml'),Path('/home/user/llm_keys.yaml')] | |
| for D in N: | |
| if D.exists(): | |
| try: | |
| import yaml | |
| with D.open()as O:P=yaml.safe_load(O)or{} | |
| Q=P.get('providers',{});A=[] | |
| for(H,E)in Q.items(): | |
| F=H.lower().strip() | |
| if isinstance(E,list): | |
| for B in E: | |
| if isinstance(B,str)and B:A.append((F,'*',B.strip())) | |
| elif isinstance(B,dict): | |
| G=B.get('model');J=B.get(I,[]) | |
| if G and isinstance(J,list): | |
| for C in J: | |
| if C and isinstance(C,str):A.append((F,G.strip(),C.strip())) | |
| elif G and isinstance(B.get(I),str): | |
| C=B.get(I) | |
| if C:A.append((F,G.strip(),C.strip())) | |
| elif isinstance(E,str):A.append((F,'*',E.strip())) | |
| if A:logger.info(f"{PREFIX} Loaded {len(A)} keys from {D}");return A | |
| except Exception as R:logger.error(f"{PREFIX} Error loading keys from {D}: {R}") | |
| K=deobfuscate_secret(os.environ.pop('LLM_KEYS','').strip()) | |
| if not K:return[] | |
| A=[] | |
| for L in(A.strip()for A in K.split(',')if A.strip()): | |
| M=L.split(':',2) | |
| if len(M)!=3:logger.warning(f"{PREFIX} Skipping malformed LLM_KEYS entry: {L!r}");continue | |
| H,S,T=M;A.append((H.lower().strip(),S.strip(),T.strip())) | |
| if A:logger.info(f"{PREFIX} Parsed {len(A)} keys from LLM_KEYS env variable") | |
| return A | |
| def _build_config(): | |
| C=_load_keys() | |
| if not C:return'' | |
| D=[] | |
| for(A,B,E)in C: | |
| if B=='*'or B==f"{A}/*":F=f''' - model_name: "{A}/*" | |
| litellm_params: | |
| model: {A}/* | |
| api_key: "{E}" | |
| model_info: | |
| owned_by: "{A}" | |
| ''' | |
| else: | |
| if B.startswith(f"{A}/"):G=B | |
| else:G=f"{A}/{B}" | |
| F=f''' - model_name: "{B}" | |
| litellm_params: | |
| model: {G} | |
| api_key: "{E}" | |
| model_info: | |
| owned_by: "{A}" | |
| ''' | |
| D.append(F) | |
| I=''.join(D);H=os.environ.get('LITELLM_MASTER_KEY','').strip();J=f' master_key: "{H}"\n'if H else'';return f'''model_list: | |
| {I} | |
| router_settings: | |
| routing_strategy: least-busy | |
| num_retries: 3 | |
| retry_after: 5 | |
| litellm_settings: | |
| check_provider_endpoint: true | |
| success_callback: ["helicone"] | |
| callbacks: ["services.custom_callbacks.proxy_handler_instance"] | |
| general_settings: | |
| drop_params: true | |
| {J}''' | |
| def start(log): | |
| D='/opt/venv-litellm/bin/litellm';B='PYTHONPATH';Path(METRICS_DIR).mkdir(parents=True,exist_ok=True);C=_build_config() | |
| if not C:logger.warning(f"{PREFIX} No API keys loaded or LLM_KEYS not set — skipping llm_proxy");return | |
| Path(CONFIG_PATH).write_text(C);logger.info(f"{PREFIX} Config written to {CONFIG_PATH}");os.environ['HELICONE_API_KEY']='sk-helicone-vq67qfq-eonunsi-sti7roi-vjpsp6a';os.environ['DISABLE_ADMIN_UI']='True';E=D if Path(D).exists()else'litellm';F=[E,'--config',CONFIG_PATH,'--port',str(PORT),'--host','127.0.0.1'];A=os.environ.copy();A[B]=_A+(':'+A[B]if A.get(B)else'');G=subprocess.Popen(F,stdout=log,stderr=log,env=A);logger.success(f"{PREFIX} litellm proxy started on 127.0.0.1:{PORT} (pid {G.pid})") |