{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "c9512259-712a-490e-95c8-e1e5e2e05338", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: hf_transfer in /usr/local/lib/python3.11/dist-packages (0.1.9)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\u001b[33m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m26.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n" ] } ], "source": [ "!pip install hf_transfer" ] }, { "cell_type": "code", "execution_count": null, "id": "7281e752-75f0-4eaf-af3a-c92683496a74", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO 05-02 22:01:02 [utils.py:233] non-default args: {'tensor_parallel_size': 8, 'disable_log_stats': True, 'model': 'meta-llama/Meta-Llama-3.1-70B-Instruct'}\n", "INFO 05-02 22:01:06 [model.py:555] Resolved architecture: LlamaForCausalLM\n", "INFO 05-02 22:01:06 [model.py:1680] Using max model len 131072\n", "INFO 05-02 22:01:06 [scheduler.py:239] Chunked prefill is enabled with max_num_batched_tokens=16384.\n", "INFO 05-02 22:01:06 [vllm.py:840] Asynchronous scheduling is enabled.\n", "INFO 05-02 22:01:06 [kernel.py:205] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'])\n", "INFO 05-02 22:01:10 [compilation.py:303] Enabled custom fusions: allreduce_rms\n", "WARNING 05-02 22:01:15 [system_utils.py:157] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized\n", "(EngineCore pid=3440) INFO 05-02 22:01:20 [core.py:109] Initializing a V1 LLM engine (v0.20.0) with config: model='meta-llama/Meta-Llama-3.1-70B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-70B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=8, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-70B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [32, 16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto')\n", "(EngineCore pid=3440) WARNING 05-02 22:01:20 [multiproc_executor.py:1029] Reducing Torch parallelism from 112 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.\n", "(EngineCore pid=3440) INFO 05-02 22:01:20 [multiproc_executor.py:139] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.20.0.2 (local), world_size=8, local_world_size=8\n", "INFO 05-02 22:01:27 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:27 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:27 [nixl_utils.py:44] NIXL agent config is not available\n", "INFO 05-02 22:01:27 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:27 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:27 [nixl_utils.py:44] NIXL agent config is not available\n", "INFO 05-02 22:01:28 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "INFO 05-02 22:01:28 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:28 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:44] NIXL agent config is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:44] NIXL agent config is not available\n", "INFO 05-02 22:01:28 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:28 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:44] NIXL agent config is not available\n", "INFO 05-02 22:01:28 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:28 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:44] NIXL agent config is not available\n", "INFO 05-02 22:01:28 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:28 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:44] NIXL agent config is not available\n", "INFO 05-02 22:01:28 [nixl_utils.py:20] Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare memory leak in UCX when using NIXL.\n", "WARNING 05-02 22:01:28 [nixl_utils.py:34] NIXL is not available\n", "WARNING 05-02 22:01:28 [nixl_utils.py:44] NIXL agent config is not available\n", "(Worker pid=3642) INFO 05-02 22:01:28 [parallel_state.py:1402] world_size=8 rank=3 local_rank=3 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3640) INFO 05-02 22:01:28 [parallel_state.py:1402] world_size=8 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3639) INFO 05-02 22:01:29 [parallel_state.py:1402] world_size=8 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3645) INFO 05-02 22:01:31 [parallel_state.py:1402] world_size=8 rank=6 local_rank=6 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3646) INFO 05-02 22:01:31 [parallel_state.py:1402] world_size=8 rank=7 local_rank=7 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3643) INFO 05-02 22:01:31 [parallel_state.py:1402] world_size=8 rank=4 local_rank=4 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3641) INFO 05-02 22:01:31 [parallel_state.py:1402] world_size=8 rank=2 local_rank=2 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3644) INFO 05-02 22:01:31 [parallel_state.py:1402] world_size=8 rank=5 local_rank=5 distributed_init_method=tcp://127.0.0.1:57135 backend=nccl\n", "(Worker pid=3639) INFO 05-02 22:01:31 [pynccl.py:111] vLLM is using nccl==2.28.9\n", "(Worker pid=3639) INFO 05-02 22:01:34 [parallel_state.py:1715] rank 0 in world size 8 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A\n", "(Worker_TP0 pid=3639) INFO 05-02 22:01:35 [gpu_model_runner.py:4777] Starting to load model meta-llama/Meta-Llama-3.1-70B-Instruct...\n", "(Worker_TP0 pid=3639) INFO 05-02 22:01:35 [cuda.py:368] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].\n", "(Worker_TP0 pid=3639) INFO 05-02 22:01:35 [flash_attn.py:646] Using FlashAttention version 3\n" ] } ], "source": [ "from vllm import LLM\n", "llm = LLM(model=\"meta-llama/Meta-Llama-3.1-70B-Instruct\", tensor_parallel_size=8)" ] }, { "cell_type": "code", "execution_count": null, "id": "d5e41715-45af-4b50-8771-09854f0d6e7f", "metadata": {}, "outputs": [], "source": [ "import json\n", "from tqdm import tqdm\n", "import re\n", "from vllm import SamplingParams\n", "\n", "with open(\"fd_pre_texts.json\") as f:\n", " data = json.load(f)\n", "\n", "system_prompt = \"\"\"You are a veteran financial advisor with 22 years advising sophisticated clients and institutions. Rephrase the texts in a natural, authoritative, polished human voice — exactly as a seasoned expert would write client notes.\n", "\n", "Add meaningful nuance and professional insight through refined phrasing only.\n", "Strictly preserve every fact, number, percentage, date, and data point unchanged. Never add, omit, or alter anything.\n", "\n", "Write with varied sentence rhythm and fresh vocabulary for smooth, human flow. Eliminate all repetition within and across sections.\n", "Sound confidently human and expert: precise, confident, approachable.\n", "\n", "Never use these AI-sounding words or similar: demonstrated, indicating, highlighting, underscoring, showcasing, revealing, notably, crucially, suggests, illustrates, points to, testament to.\"\"\"\n", "\n", "prompts = []\n", "instrument_map = []\n", "\n", "for category, instruments in data.items():\n", " for inst_key, row in instruments.items():\n", " user_prompt = f\"\"\"Instrument: {row['InstrumentKey']}\n", "Overall text:\n", "{row.get('buckler_pretext','').strip()}\n", "Performance text:\n", "{row.get('performance_pretext','').strip()}\n", "Risk text:\n", "{row.get('risk_pretext','').strip()}\n", "Management text:\n", "{row.get('management_pretext','').strip()}\n", "Rephrase with professional nuance.\n", "Output EXACTLY this format:\n", "Rephrased Overall:\n", "[text]\n", "Rephrased Performance:\n", "[text]\n", "Rephrased Risk:\n", "[text]\n", "Rephrased Management:\n", "[text]\"\"\"\n", " prompts.append([{\"role\": \"system\", \"content\": system_prompt}, {\"role\": \"user\", \"content\": user_prompt}])\n", " instrument_map.append((category, inst_key))\n", "\n", "sampling_params = SamplingParams(temperature=0.3, max_tokens=2048)\n", "chunk_size = 150\n", "enhanced_texts = []\n", "\n", "for i in tqdm(range(0, len(prompts), chunk_size)):\n", " chunk = prompts[i:i+chunk_size]\n", " outputs = llm.chat(chunk, sampling_params)\n", " enhanced_texts.extend([out.outputs[0].text for out in outputs])\n", "\n", "def parse_sections(text):\n", " sections = {}\n", " for key in [\"Overall\", \"Performance\", \"Risk\", \"Management\"]:\n", " match = re.search(rf\"Rephrased {key}:\\s*(.*?)(?=\\nRephrased \\w+|$)\", text, re.DOTALL)\n", " sections[key.lower()] = match.group(1).strip() if match else \"\"\n", " return sections\n", "\n", "output_data = {}\n", "idx = 0\n", "for category, inst_key in instrument_map:\n", " if category not in output_data:\n", " output_data[category] = {}\n", " output_data[category][inst_key] = {\n", " \"InstrumentKey\": inst_key,\n", " **parse_sections(enhanced_texts[idx])\n", " }\n", " idx += 1\n", "\n", "with open(\"final_fundata.json\", \"w\") as f:\n", " json.dump(output_data, f, indent=2)\n", "\n", "print(f\"Saved {len(instrument_map)} records to final_texts_fundata.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "49ad0fe4-888e-41a0-8425-e65bc5ab4a45", "metadata": {}, "outputs": [], "source": [ "import json\n", "from tqdm import tqdm\n", "import re\n", "from vllm import SamplingParams\n", "\n", "with open(\"pre_texts_eod.json\") as f:\n", " data = json.load(f)\n", "\n", "system_prompt = \"\"\"You are a veteran financial advisor with 22 years advising sophisticated clients and institutions. Rephrase the texts in a natural, authoritative, polished human voice — exactly as a seasoned expert would write client notes.\n", "Add meaningful nuance and professional insight through refined phrasing only.\n", "Strictly preserve every fact, number, percentage, date, and data point unchanged. Never add, omit, or alter anything.\n", "Write with varied sentence rhythm and fresh vocabulary for smooth, human flow. Eliminate all repetition within and across sections.\n", "Sound confidently human and expert: precise, confident, approachable.\n", "Never use these AI-sounding words or similar: demonstrated, indicating, highlighting, underscoring, showcasing, revealing, notably, crucially, suggests, illustrates, points to, testament to.\"\"\"\n", "\n", "prompts = []\n", "instrument_map = []\n", "\n", "for category, instruments in data.items():\n", " for inst_key, row in instruments.items():\n", " user_prompt = f\"\"\"Symbol: {inst_key}\n", "Overall text:\n", "{row.get('buckler_pretext','').strip()}\n", "Performance text:\n", "{row.get('performance_pretext','').strip()}\n", "Risk text:\n", "{row.get('risk_pretext','').strip()}\n", "Management text:\n", "{row.get('management_pretext','').strip()}\n", "\n", "Rephrase with professional nuance.\n", "\n", "Output EXACTLY this format:\n", "Rephrased Overall:\n", "[text]\n", "Rephrased Performance:\n", "[text]\n", "Rephrased Risk:\n", "[text]\n", "Rephrased Management:\n", "[text]\"\"\"\n", "\n", " prompts.append([{\"role\": \"system\", \"content\": system_prompt}, {\"role\": \"user\", \"content\": user_prompt}])\n", " instrument_map.append((category, inst_key))\n", "\n", "sampling_params = SamplingParams(temperature=0.3, max_tokens=2048)\n", "chunk_size = 150\n", "enhanced_texts = []\n", "\n", "for i in tqdm(range(0, len(prompts), chunk_size)):\n", " chunk = prompts[i:i+chunk_size]\n", " outputs = llm.chat(chunk, sampling_params)\n", " enhanced_texts.extend([out.outputs[0].text for out in outputs])\n", "\n", "def parse_sections(text):\n", " sections = {}\n", " for key in [\"Overall\", \"Performance\", \"Risk\", \"Management\"]:\n", " match = re.search(rf\"Rephrased {key}:\\s*(.*?)(?=\\nRephrased \\w+|$)\", text, re.DOTALL)\n", " sections[key.lower()] = match.group(1).strip() if match else \"\"\n", " return sections\n", "\n", "output_data = {}\n", "idx = 0\n", "for category, inst_key in instrument_map:\n", " if category not in output_data:\n", " output_data[category] = {}\n", " output_data[category][inst_key] = {\n", " \"symbol\": inst_key,\n", " **parse_sections(enhanced_texts[idx])\n", " }\n", " idx += 1\n", "\n", "with open(\"final_texts_EOD.json\", \"w\") as f:\n", " json.dump(output_data, f, indent=2)\n", "\n", "print(f\"Saved {len(instrument_map)} records to final_texts_EOD.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "be369250-7af0-4a3d-b84c-35c5ca09c9db", "metadata": {}, "outputs": [], "source": [ "import json\n", "from tqdm import tqdm\n", "import re\n", "from vllm import SamplingParams\n", "\n", "with open(\"eq_pre_text.json\") as f:\n", " data = json.load(f)\n", "\n", "system_prompt = \"\"\"You are a veteran financial advisor with 22 years advising sophisticated clients and institutions. Rephrase the texts in a natural, authoritative, polished human voice — exactly as a seasoned expert would write client notes.\n", "Add meaningful nuance and professional insight through refined phrasing only.\n", "Strictly preserve every fact, number, percentage, date, and data point unchanged. Never add, omit, or alter anything.\n", "Write with varied sentence rhythm and fresh vocabulary for smooth, human flow. Eliminate all repetition within and across sections.\n", "Sound confidently human and expert: precise, confident, approachable.\n", "Never use these AI-sounding words or similar: demonstrated, indicating, highlighting, underscoring, showcasing, revealing, notably, crucially, suggests, illustrates, points to, testament to.\"\"\"\n", "\n", "prompts = []\n", "symbol_list = []\n", "for symbol, row in data.items():\n", " user_prompt = f\"\"\"Symbol: {symbol}\n", "Performance text:\n", "{row.get('performance','').strip()}\n", "Valuation text:\n", "{row.get('valuation','').strip()}\n", "Financial text:\n", "{row.get('financial','').strip()}\n", "Growth text:\n", "{row.get('growth','').strip()}\n", "Buckler text:\n", "{row.get('buckler','').strip()}\n", "\n", "Rephrase with professional nuance.\n", "Output EXACTLY this format:\n", "Rephrased Performance:\n", "[text]\n", "Rephrased Valuation:\n", "[text]\n", "Rephrased Financial:\n", "[text]\n", "Rephrased Growth:\n", "[text]\n", "Rephrased Buckler:\n", "[text]\"\"\"\n", " prompts.append([{\"role\": \"system\", \"content\": system_prompt}, {\"role\": \"user\", \"content\": user_prompt}])\n", " symbol_list.append(symbol)\n", "\n", "sampling_params = SamplingParams(temperature=0.3, max_tokens=2048)\n", "chunk_size = 150\n", "enhanced_texts = []\n", "for i in tqdm(range(0, len(prompts), chunk_size)):\n", " chunk = prompts[i:i+chunk_size]\n", " outputs = llm.chat(chunk, sampling_params)\n", " enhanced_texts.extend([out.outputs[0].text for out in outputs])\n", "\n", "def parse_sections(text):\n", " sections = {}\n", " for key in [\"Performance\", \"Valuation\", \"Financial\", \"Growth\", \"Buckler\"]:\n", " match = re.search(rf\"Rephrased {key}:\\s*(.*?)(?=\\nRephrased \\w+|$)\", text, re.DOTALL)\n", " sections[key.lower()] = match.group(1).strip() if match else \"\"\n", " return sections\n", "\n", "output_data = {}\n", "for idx, symbol in enumerate(symbol_list):\n", " output_data[symbol] = {\n", " \"symbol\": symbol,\n", " **parse_sections(enhanced_texts[idx])\n", " }\n", "\n", "with open(\"final_equity_text.json\", \"w\") as f:\n", " json.dump(output_data, f, indent=2)\n", "print(f\"Saved {len(symbol_list)} records to final_equity_text.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3f941927-d921-4fd6-bbd5-b63242e1493c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8b729592-a944-4197-81a9-9f7db72e1c67", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.10" } }, "nbformat": 4, "nbformat_minor": 5 }