Add files using upload-large-folder tool

emissary-ml/llm-scripts/fine-tuning/llama3/gpu_monitoring.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+GPU Memory Monitoring Script for Model Parallelization Experiments
+"""
+
+import subprocess
+import time
+import csv
+import datetime
+import argparse
+
+
+def get_gpu_memory_info():
+    """Get current GPU memory usage using nvidia-smi"""
+    try:
+        result = subprocess.run(
+            ['nvidia-smi', '--query-gpu=index,name,memory.used,memory.total,utilization.gpu', 
+             '--format=csv,noheader,nounits'],
+            capture_output=True, text=True, check=True
+        )
+        
+        gpu_info = []
+        for line in result.stdout.strip().split('\n'):
+            parts = line.split(', ')
+            gpu_info.append({
+                'index': int(parts[0]),
+                'name': parts[1],
+                'memory_used_mb': int(parts[2]),
+                'memory_total_mb': int(parts[3]),
+                'gpu_utilization': int(parts[4])
+            })
+        return gpu_info
+    except Exception as e:
+        print(f"Error getting GPU info: {e}")
+        return []
+
+def monitor_gpus(output_file, interval=5, experiment_name=""):
+    """Monitor GPU memory usage and save to CSV"""
+    
+    with open(output_file, 'w', newline='') as csvfile:
+        fieldnames = ['timestamp', 'experiment', 'gpu_index', 'gpu_name', 
+                     'memory_used_mb', 'memory_total_mb', 'memory_percent', 
+                     'gpu_utilization']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer.writeheader()
+        
+        print(f"Starting GPU monitoring for experiment: {experiment_name}")
+        print(f"Writing to: {output_file}")
+        print("Press Ctrl+C to stop monitoring\n")
+        
+        try:
+            while True:
+                timestamp = datetime.datetime.now().isoformat()
+                gpu_infos = get_gpu_memory_info()
+                
+                for gpu in gpu_infos:
+                    memory_percent = (gpu['memory_used_mb'] / gpu['memory_total_mb']) * 100
+                    
+                    writer.writerow({
+                        'timestamp': timestamp,
+                        'experiment': experiment_name,
+                        'gpu_index': gpu['index'],
+                        'gpu_name': gpu['name'],
+                        'memory_used_mb': gpu['memory_used_mb'],
+                        'memory_total_mb': gpu['memory_total_mb'],
+                        'memory_percent': f"{memory_percent:.2f}",
+                        'gpu_utilization': gpu['gpu_utilization']
+                    })
+                    
+                    print(f"GPU {gpu['index']}: {gpu['memory_used_mb']}/{gpu['memory_total_mb']} MB "
+                          f"({memory_percent:.1f}%) | Util: {gpu['gpu_utilization']}%")
+                
+                print("-" * 80)
+                csvfile.flush()
+                time.sleep(interval)
+                
+        except KeyboardInterrupt:
+            print("\nMonitoring stopped.")
+            
+def analyze_log(log_file):
+    """Analyze the monitoring log and produce summary statistics"""
+    data = []
+    with open(log_file, 'r') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            row['memory_used_mb'] = int(row['memory_used_mb'])
+            row['memory_total_mb'] = int(row['memory_total_mb'])
+            row['memory_percent'] = float(row['memory_percent'])
+            row['gpu_utilization'] = int(row['gpu_utilization'])
+            data.append(row)
+    
+    if not data:
+        print("No data found in log file")
+        return
+    
+    # Group by GPU
+    gpus = {}
+    for row in data:
+        gpu_idx = row['gpu_index']
+        if gpu_idx not in gpus:
+            gpus[gpu_idx] = []
+        gpus[gpu_idx].append(row)
+    
+    print(f"\nAnalysis of {log_file}:")
+    print("=" * 80)
+    
+    for gpu_idx, gpu_data in sorted(gpus.items()):
+        memory_used = [d['memory_used_mb'] for d in gpu_data]
+        memory_percent = [d['memory_percent'] for d in gpu_data]
+        gpu_util = [d['gpu_utilization'] for d in gpu_data]
+        
+        print(f"\nGPU {gpu_idx} ({gpu_data[0]['gpu_name']}):")
+        print(f"  Memory - Max: {max(memory_used)} MB ({max(memory_percent):.1f}%)")
+        print(f"  Memory - Avg: {sum(memory_used)/len(memory_used):.0f} MB ({sum(memory_percent)/len(memory_percent):.1f}%)")
+        print(f"  GPU Util - Max: {max(gpu_util)}%")
+        print(f"  GPU Util - Avg: {sum(gpu_util)/len(gpu_util):.1f}%")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='GPU Memory Monitor for ML Experiments')
+    parser.add_argument('--output', '-o', default='gpu_monitor.csv', 
+                       help='Output CSV file')
+    parser.add_argument('--interval', '-i', type=int, default=5,
+                       help='Monitoring interval in seconds')
+    parser.add_argument('--experiment', '-e', default='',
+                       help='Experiment name/description')
+    parser.add_argument('--analyze', '-a', 
+                       help='Analyze existing log file instead of monitoring')
+    
+    args = parser.parse_args()
+    
+    if args.analyze:
+        analyze_log(args.analyze)
+    else:
+        monitor_gpus(args.output, args.interval, args.experiment)

emissary-ml/llm-scripts/fine-tuning/llama3/test_script.py

@@ -0,0 +1 @@
+test_functions = {}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/labextensions/@jupyter-notebook/lab-extension/package.json

@@ -0,0 +1,109 @@
+{
+  "name": "@jupyter-notebook/lab-extension",
+  "version": "7.4.2",
+  "description": "Jupyter Notebook - Lab Extension",
+  "homepage": "https://github.com/jupyter/notebook",
+  "bugs": {
+    "url": "https://github.com/jupyter/notebook/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jupyter/notebook.git"
+  },
+  "license": "BSD-3-Clause",
+  "author": "Project Jupyter",
+  "sideEffects": [
+    "style/**/*.css",
+    "style/index.js"
+  ],
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "style": "style/index.css",
+  "directories": {
+    "lib": "lib/"
+  },
+  "files": [
+    "lib/*.d.ts",
+    "lib/*.js.map",
+    "lib/*.js",
+    "schema/*.json",
+    "style/index.js"
+  ],
+  "scripts": {
+    "build": "jlpm run build:labextension:dev",
+    "build:labextension": "jupyter labextension build .",
+    "build:labextension:dev": "jupyter labextension build --development True .",
+    "build:lib": "tsc -b",
+    "build:prod": "jlpm run build:labextension",
+    "clean": "jlpm run clean:lib && jlpm run clean:labextension",
+    "clean:labextension": "rimraf ../../notebook/labextension",
+    "clean:lib": "rimraf lib tsconfig.tsbuildinfo",
+    "watch": "run-p watch:src watch:labextension",
+    "watch:labextension": "jupyter labextension watch .",
+    "watch:src": "tsc -w"
+  },
+  "dependencies": {
+    "@jupyter-notebook/application": "^7.4.2",
+    "@jupyterlab/application": "~4.4.2",
+    "@jupyterlab/apputils": "~4.5.2",
+    "@jupyterlab/coreutils": "~6.4.2",
+    "@jupyterlab/docregistry": "~4.4.2",
+    "@jupyterlab/notebook": "~4.4.2",
+    "@jupyterlab/translation": "~4.4.2",
+    "@jupyterlab/ui-components": "~4.4.2",
+    "@lumino/commands": "^2.3.2",
+    "@lumino/disposable": "^2.1.4"
+  },
+  "devDependencies": {
+    "@jupyterlab/builder": "~4.4.2",
+    "rimraf": "^3.0.2",
+    "typescript": "~5.5.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "jupyterlab": {
+    "extension": true,
+    "outputDir": "../../notebook/labextension",
+    "schemaDir": "schema",
+    "_build": {
+      "load": "static/remoteEntry.a7f6d6f20e99b7f1c604.js",
+      "extension": "./extension",
+      "style": "./style"
+    }
+  },
+  "nx": {
+    "targets": {
+      "build:labextension:dev": {
+        "dependsOn": [
+          "^build:lib",
+          "build:lib"
+        ],
+        "outputs": [
+          "{workspaceRoot}/notebook/labextension",
+          "{workspaceRoot}/notebook/labextension/build_log.json"
+        ]
+      },
+      "build:labextension": {
+        "dependsOn": [
+          "^build:lib",
+          "build:lib"
+        ],
+        "outputs": [
+          "{workspaceRoot}/notebook/labextension"
+        ]
+      },
+      "build": {
+        "dependsOn": [
+          "build:labextension:dev"
+        ]
+      },
+      "build:prod": {
+        "dependsOn": [
+          "build:labextension"
+        ]
+      }
+    }
+  },
+  "styleModule": "style/index.js"
+}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/labextensions/@jupyter-notebook/lab-extension/static/style.js

@@ -0,0 +1,4 @@
+/* This is a generated file of CSS imports */
+/* It was generated by @jupyterlab/builder in Build.ensureAssets() */
+
+import '@jupyter-notebook/lab-extension/style/index.js';

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/labextensions/jupyterlab_pygments/package.json

@@ -0,0 +1,205 @@
+{
+  "name": "jupyterlab_pygments",
+  "version": "0.3.0",
+  "description": "Pygments theme using JupyterLab CSS variables",
+  "keywords": [
+    "jupyter",
+    "jupyterlab",
+    "jupyterlab-extension"
+  ],
+  "homepage": "https://github.com/jupyterlab/jupyterlab_pygments",
+  "bugs": {
+    "url": "https://github.com/jupyterlab/jupyterlab_pygments/issues"
+  },
+  "license": "BSD-3-Clause",
+  "author": {
+    "name": "Jupyter Development Team",
+    "email": "jupyter@googlegroups.com"
+  },
+  "files": [
+    "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}",
+    "style/**/*.{css,js,eot,gif,html,jpg,json,png,svg,woff2,ttf}",
+    "style/index.js"
+  ],
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "style": "style/index.css",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jupyterlab/jupyterlab_pygments.git"
+  },
+  "scripts": {
+    "build": "jlpm build:css && jlpm build:lib && jlpm build:labextension:dev",
+    "build:css": "python generate_css.py",
+    "build:labextension": "jupyter labextension build .",
+    "build:labextension:dev": "jupyter labextension build --development True .",
+    "build:lib": "tsc",
+    "build:prod": "jlpm clean && jlpm build:css && jlpm build:lib && jlpm build:labextension",
+    "clean": "jlpm clean:lib",
+    "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache",
+    "clean:labextension": "rimraf jupyterlab_pygments/labextension",
+    "clean:lib": "rimraf lib tsconfig.tsbuildinfo style/base.css",
+    "clean:lintcache": "rimraf .eslintcache .stylelintcache",
+    "eslint": "jlpm eslint:check --fix",
+    "eslint:check": "eslint . --cache --ext .ts,.tsx",
+    "install:extension": "jlpm build",
+    "lint": "jlpm stylelint && jlpm prettier && jlpm eslint",
+    "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check",
+    "prettier": "jlpm prettier:base --write --list-different",
+    "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"",
+    "prettier:check": "jlpm prettier:base --check",
+    "stylelint": "jlpm stylelint:check --fix",
+    "stylelint:check": "stylelint --cache \"style/**/*.css\"",
+    "watch": "run-p watch:src watch:labextension",
+    "watch:labextension": "jupyter labextension watch .",
+    "watch:src": "tsc -w"
+  },
+  "dependencies": {
+    "@jupyterlab/application": "^4.0.8",
+    "@types/node": "^20.9.0"
+  },
+  "devDependencies": {
+    "@jupyterlab/builder": "^4.0.0",
+    "@types/json-schema": "^7.0.11",
+    "@types/react": "^18.0.26",
+    "@types/react-addons-linked-state-mixin": "^0.14.22",
+    "@typescript-eslint/eslint-plugin": "^6.1.0",
+    "@typescript-eslint/parser": "^6.1.0",
+    "css-loader": "^6.7.1",
+    "eslint": "^8.36.0",
+    "eslint-config-prettier": "^8.8.0",
+    "eslint-plugin-prettier": "^5.0.0",
+    "npm-run-all": "^4.1.5",
+    "prettier": "3.0.3",
+    "rimraf": "^5.0.5",
+    "source-map-loader": "^1.0.2",
+    "style-loader": "^3.3.1",
+    "stylelint": "^15.10.1",
+    "stylelint-config-prettier": "^9.0.3",
+    "stylelint-config-recommended": "^13.0.0",
+    "stylelint-config-standard": "^34.0.0",
+    "stylelint-csstree-validator": "^3.0.0",
+    "stylelint-prettier": "^4.0.0",
+    "typescript": "~5.0.2",
+    "yjs": "^13.5.40"
+  },
+  "sideEffects": [
+    "style/*.css",
+    "style/index.js"
+  ],
+  "styleModule": "style/index.js",
+  "publishConfig": {
+    "access": "public"
+  },
+  "jupyterlab": {
+    "extension": true,
+    "outputDir": "jupyterlab_pygments/labextension",
+    "_build": {
+      "load": "static/remoteEntry.5cbb9d2323598fbda535.js",
+      "extension": "./extension",
+      "style": "./style"
+    }
+  },
+  "jupyter-releaser": {
+    "hooks": {
+      "before-build-npm": [
+        "python -m pip install jupyterlab~=3.1",
+        "jlpm"
+      ],
+      "before-build-python": [
+        "jlpm clean:all"
+      ]
+    }
+  },
+  "eslintConfig": {
+    "extends": [
+      "eslint:recommended",
+      "plugin:@typescript-eslint/eslint-recommended",
+      "plugin:@typescript-eslint/recommended",
+      "plugin:prettier/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+      "project": "tsconfig.json",
+      "sourceType": "module"
+    },
+    "plugins": [
+      "@typescript-eslint"
+    ],
+    "rules": {
+      "@typescript-eslint/naming-convention": [
+        "error",
+        {
+          "selector": "interface",
+          "format": [
+            "PascalCase"
+          ],
+          "custom": {
+            "regex": "^I[A-Z]",
+            "match": true
+          }
+        }
+      ],
+      "@typescript-eslint/no-unused-vars": [
+        "warn",
+        {
+          "args": "none"
+        }
+      ],
+      "@typescript-eslint/no-explicit-any": "off",
+      "@typescript-eslint/no-namespace": "off",
+      "@typescript-eslint/no-use-before-define": "off",
+      "@typescript-eslint/quotes": [
+        "error",
+        "single",
+        {
+          "avoidEscape": true,
+          "allowTemplateLiterals": false
+        }
+      ],
+      "curly": [
+        "error",
+        "all"
+      ],
+      "eqeqeq": "error",
+      "prefer-arrow-callback": "error"
+    }
+  },
+  "eslintIgnore": [
+    "node_modules",
+    "dist",
+    "coverage",
+    "**/*.d.ts"
+  ],
+  "prettier": {
+    "singleQuote": true,
+    "trailingComma": "none",
+    "arrowParens": "avoid",
+    "endOfLine": "auto",
+    "overrides": [
+      {
+        "files": "package.json",
+        "options": {
+          "tabWidth": 4
+        }
+      }
+    ]
+  },
+  "stylelint": {
+    "extends": [
+      "stylelint-config-recommended",
+      "stylelint-config-standard",
+      "stylelint-prettier/recommended"
+    ],
+    "plugins": [
+      "stylelint-csstree-validator"
+    ],
+    "rules": {
+      "csstree/validator": true,
+      "property-no-vendor-prefix": null,
+      "selector-class-pattern": "^([a-z][A-z\\d]*)(-[A-z\\d]+)*$",
+      "selector-no-vendor-prefix": null,
+      "value-no-vendor-prefix": null
+    }
+  }
+}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/base/cell_id_anchor.j2

@@ -0,0 +1,5 @@
+{%- macro cell_id_anchor(cell) -%}
+    {% if cell.id | length > 0 -%}
+        id="{{ ('cell-id=' ~ cell.id) | escape_html -}}"
+    {%- endif %}
+{%- endmacro %}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/base/display_priority.j2

@@ -0,0 +1,49 @@
+{%- extends 'base/null.j2' -%}
+
+{#display data priority#}
+
+
+{%- block data_priority scoped -%}
+    {%- for type in output.data | filter_data_type -%}
+        {%- if type == 'application/pdf' -%}
+            {%- block data_pdf -%}
+            {%- endblock -%}
+        {%- elif type == 'image/svg+xml' -%}
+            {%- block data_svg -%}
+            {%- endblock -%}
+        {%- elif type == 'image/png' -%}
+            {%- block data_png -%}
+            {%- endblock -%}
+        {%- elif type == 'text/html' -%}
+            {%- block data_html -%}
+            {%- endblock -%}
+        {%- elif type == 'text/markdown' -%}
+            {%- block data_markdown -%}
+            {%- endblock -%}
+        {%- elif type == 'image/jpeg' -%}
+            {%- block data_jpg -%}
+            {%- endblock -%}
+        {%- elif type == 'text/plain' -%}
+            {%- block data_text -%}
+            {%- endblock -%}
+        {%- elif type == 'text/latex' -%}
+            {%- block data_latex -%}
+            {%- endblock -%}
+        {%- elif type == 'text/vnd.mermaid' -%}
+            {%- block data_mermaid -%}
+            {%- endblock -%}
+        {%- elif type == 'application/javascript' -%}
+            {%- block data_javascript -%}
+            {%- endblock -%}
+        {%- elif type == 'application/vnd.jupyter.widget-view+json' -%}
+            {%- block data_widget_view -%}
+            {%- endblock -%}
+        {%- elif type == resources.output_mimetype -%}
+            {%- block data_native -%}
+            {%- endblock -%}
+        {%- else -%}
+            {%- block data_other -%}
+            {%- endblock -%}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endblock data_priority -%}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/base/null.j2

@@ -0,0 +1,111 @@
+{#
+
+DO NOT USE THIS AS A BASE,
+IF YOU ARE COPY AND PASTING THIS FILE
+YOU ARE PROBABLY DOING THINGS INCORRECTLY.
+
+Null template, does nothing except defining a basic structure
+To layout the different blocks of a notebook.
+
+Subtemplates can override blocks to define their custom representation.
+
+If one of the block you do overwrite is not a leaf block, consider
+calling super.
+
+{%- block nonLeafBlock -%}
+    #add stuff at beginning
+    {{ super() }}
+    #add stuff at end
+{%- endblock nonLeafBlock -%}
+
+consider calling super even if it is a leaf block, we might insert more blocks later.
+
+#}
+{%- block header -%}
+{%- endblock header -%}
+{%- block body -%}
+    {%- block body_header -%}
+    {%- endblock body_header -%}
+    {%- block body_loop -%}
+        {%- for cell in nb.cells -%}
+            {%- block any_cell scoped -%}
+                {%- if cell.cell_type == 'code'-%}
+                    {%- if resources.global_content_filter.include_code -%}
+                    {%- block codecell scoped -%}
+                        {%- if resources.global_content_filter.include_input and not cell.metadata.get("transient",{}).get("remove_source", false) -%}
+                            {%- block input_group -%}
+                            {%- if resources.global_content_filter.include_input_prompt -%}
+                                {%- block in_prompt -%}{%- endblock in_prompt -%}
+                            {%- endif -%}
+                                {%- block input -%}{%- endblock input -%}
+                            {%- endblock input_group -%}
+                        {%- endif -%}
+                        {%- if cell.outputs and resources.global_content_filter.include_output -%}
+                            {%- block output_group -%}
+                                {%- if resources.global_content_filter.include_output_prompt -%}
+                                    {%- block output_prompt -%}{%- endblock output_prompt -%}
+                                {%- endif -%}
+                                {%- block outputs scoped -%}
+                                    {%- for output in cell.outputs -%}
+                                        {%- block output scoped -%}
+                                            {%- if output.output_type == 'execute_result' -%}
+                                                {%- block execute_result scoped -%}{%- endblock execute_result -%}
+                                            {%- elif output.output_type == 'stream' -%}
+                                                {%- block stream scoped -%}
+                                                    {%- if output.name == 'stdout' -%}
+                                                        {%- block stream_stdout scoped -%}
+                                                        {%- endblock stream_stdout -%}
+                                                    {%- elif output.name == 'stderr' -%}
+                                                        {%- block stream_stderr scoped -%}
+                                                        {%- endblock stream_stderr -%}
+                                                    {%- elif output.name == 'stdin' -%}
+                                                        {%- block stream_stdin scoped -%}
+                                                        {%- endblock stream_stdin -%}
+                                                    {%- endif -%}
+                                                {%- endblock stream -%}
+                                            {%- elif output.output_type == 'display_data' -%}
+                                                {%- block display_data scoped -%}
+                                                    {%- block data_priority scoped -%}
+                                                    {%- endblock data_priority -%}
+                                                {%- endblock display_data -%}
+                                            {%- elif output.output_type == 'error' -%}
+                                                {%- block error scoped -%}
+                                                {%- for line in output.traceback -%}
+                                                    {%- block traceback_line scoped -%}{%- endblock traceback_line -%}
+                                                {%- endfor -%}
+                                                {%- endblock error -%}
+                                            {%- endif -%}
+                                        {%- endblock output -%}
+                                    {%- endfor -%}
+                                {%- endblock outputs -%}
+                            {%- endblock output_group -%}
+                        {%- endif -%}
+                    {%- endblock codecell -%}
+                    {%- endif -%}
+                {%- elif cell.cell_type in ['markdown'] -%}
+                    {%- if resources.global_content_filter.include_markdown and not cell.metadata.get("transient",{}).get("remove_source", false) -%}
+                        {%- block markdowncell scoped-%} {%- endblock markdowncell -%}
+                    {%- endif -%}
+                {%- elif cell.cell_type in ['raw'] -%}
+                    {%- if resources.global_content_filter.include_raw and not cell.metadata.get("transient",{}).get("remove_source", false) -%}
+                        {%- block rawcell scoped -%}
+                        {%- if cell.metadata.get('raw_mimetype', '').lower() in resources.get('raw_mimetypes', ['']) -%}
+                        {{ cell.source }}
+                        {%- endif -%}
+                        {%- endblock rawcell -%}
+                    {%- endif -%}
+                {%- else -%}
+                    {%- if resources.global_content_filter.include_unknown and not cell.metadata.get("transient",{}).get("remove_source", false) -%}
+                        {%- block unknowncell scoped-%}
+                        {%- endblock unknowncell -%}
+                    {%- endif -%}
+                {%- endif -%}
+            {%- endblock any_cell -%}
+        {%- endfor -%}
+    {%- endblock body_loop -%}
+    {%- block body_footer -%}
+    {%- endblock body_footer -%}
+{%- endblock body -%}
+
+{%- block footer -%}
+{%- endblock footer -%}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/reveal/index.html.j2

@@ -0,0 +1,194 @@
+{%- extends 'base.html.j2' -%}
+{% from 'mathjax.html.j2' import mathjax %}
+{% from 'jupyter_widgets.html.j2' import jupyter_widgets %}
+
+{% set reveal_url_prefix = resources.reveal.url_prefix | default('https://unpkg.com/reveal.js@4.0.2', true) %}
+{% set reveal_theme = resources.reveal.theme | default('white', true) %}
+{% set reveal_transition = resources.reveal.transition | default('slide', true) %}
+{% set reveal_number = resources.reveal.number | default('', true) %}
+{% set reveal_width = resources.reveal.width | default('960', true) %}
+{% set reveal_height = resources.reveal.height | default('700', true) %}
+{% set reveal_scroll = resources.reveal.scroll | default(false, true) | json_dumps %}
+
+{%- block header -%}
+<!DOCTYPE html>
+<html lang="{{ resources.language_code }}">
+<head>
+
+{%- block html_head -%}
+<meta charset="utf-8" />
+<meta http-equiv="X-UA-Compatible" content="chrome=1" />
+
+<meta name="apple-mobile-web-app-capable" content="yes" />
+<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
+
+{% set nb_title = nb.metadata.get('title', resources['metadata']['name']) | escape_html_keep_quotes %}
+<title>{{nb_title}} slides</title>
+
+{%- block html_head_js -%}
+{%- block html_head_js_jquery -%}
+<script src="{{ resources.jquery_url }}"></script>
+{%- endblock html_head_js_jquery -%}
+{%- block html_head_js_requirejs -%}
+<script src="{{ resources.require_js_url }}"></script>
+{%- endblock html_head_js_requirejs -%}
+{%- block html_head_js_mermaidjs -%}
+<script type="module">
+  import mermaid from '{{ resources.mermaid_js_url }}';
+  mermaid.initialize({ startOnLoad: true });
+</script>
+{%- endblock html_head_js_mermaidjs -%}
+{%- endblock html_head_js -%}
+
+{% block jupyter_widgets %}
+  {%- if "widgets" in nb.metadata -%}
+    {{ jupyter_widgets(resources.jupyter_widgets_base_url, resources.html_manager_semver_range, resources.widget_renderer_url) }}
+  {%- endif -%}
+{% endblock jupyter_widgets %}
+
+<!-- General and theme style sheets -->
+<link rel="stylesheet" href="{{ reveal_url_prefix }}/dist/reveal.css">
+
+<!-- If the query includes 'print-pdf', include the PDF print sheet -->
+<script>
+if( window.location.search.match( /print-pdf/gi ) ) {
+        var link = document.createElement( 'link' );
+        link.rel = 'stylesheet';
+        link.type = 'text/css';
+        document.getElementsByTagName( 'head' )[0].appendChild( link );
+}
+</script>
+
+{% for css in resources.inlining.css -%}
+  <style type="text/css">
+    {{ css }}
+  </style>
+{% endfor %}
+
+{% block notebook_css %}
+{{ resources.include_css("static/index.css") }}
+{% if resources.theme == 'dark' %}
+    {{ resources.include_css("static/theme-dark.css") }}
+{% else %}
+    {{ resources.include_css("static/theme-light.css") }}
+{% endif %}
+<style type="text/css">
+a.anchor-link {
+   display: none;
+}
+.highlight  {
+    margin: 0.4em;
+}
+.jp-Notebook {
+    padding: 0;
+}
+:root {
+    --jp-ui-font-size1: 20px;       /* instead of 14px */
+    --jp-content-font-size1: 20px;  /* instead of 14px */
+    --jp-code-font-size: 19px;      /* instead of 13px */
+    --jp-cell-prompt-width: 110px;  /* instead of 64px */
+}
+@media print {
+  body {
+    margin: 0;
+  }
+}
+</style>
+
+{{ resources.include_css("static/custom_reveal.css") }}
+
+{% endblock notebook_css %}
+
+{%- block html_head_js_mathjax -%}
+{{ mathjax(resources.mathjax_url) }}
+{%- endblock html_head_js_mathjax -%}
+
+{%- block html_head_css -%}
+{%- endblock html_head_css -%}
+
+{%- endblock html_head -%}
+
+<!-- Reveal Theme -->
+<link rel="stylesheet" href="{{ reveal_url_prefix }}/dist/theme/{{reveal_theme}}.css" id="theme">
+
+</head>
+{% endblock header%}
+
+{%- block body_header -%}
+{% if resources.theme == 'dark' %}
+<body class="jp-Notebook" data-jp-theme-light="false" data-jp-theme-name="JupyterLab Dark">
+{% else %}
+<body class="jp-Notebook" data-jp-theme-light="true" data-jp-theme-name="JupyterLab Light">
+{% endif %}
+<main>
+<div class="reveal">
+<div class="slides">
+{%- endblock body_header -%}
+
+{% block body_footer %}
+</div>
+</div>
+</main>
+</body>
+{% endblock body_footer %}
+
+{% block footer %}
+{{ super() }}
+
+{% block footer_js %}
+<script>
+require(
+    {
+      // it makes sense to wait a little bit when you are loading
+      // reveal from a cdn in a slow connection environment
+      waitSeconds: 15
+    },
+    [
+      "{{ reveal_url_prefix }}/dist/reveal.js",
+      "{{ reveal_url_prefix }}/plugin/notes/notes.js"
+    ],
+
+    function(Reveal, RevealNotes){
+        // Full list of configuration options available here: https://github.com/hakimel/reveal.js#configuration
+        Reveal.initialize({
+            controls: true,
+            progress: true,
+            history: true,
+            transition: "{{reveal_transition}}",
+            slideNumber: "{{reveal_number}}",
+            plugins: [RevealNotes],
+            width: {{reveal_width}},
+			      height: {{reveal_height}},
+
+        });
+
+        var update = function(event){
+          if(MathJax.Hub.getAllJax(Reveal.getCurrentSlide())){
+            MathJax.Hub.Rerender(Reveal.getCurrentSlide());
+          }
+        };
+
+        Reveal.addEventListener('slidechanged', update);
+
+        function setScrollingSlide() {
+            var scroll = {{ reveal_scroll }}
+            if (scroll === true) {
+              var h = $('.reveal').height() * 0.95;
+              $('section.present').find('section')
+                .filter(function() {
+                  return $(this).height() > h;
+                })
+                .css('height', 'calc(95vh)')
+                .css('overflow-y', 'scroll')
+                .css('margin-top', '20px');
+            }
+        }
+
+        // check and set the scrolling slide every time the slide change
+        Reveal.addEventListener('slidechanged', setScrollingSlide);
+    }
+);
+</script>
+{% endblock footer_js %}
+</html>
+{% endblock footer %}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/reveal/static/custom_reveal.css

@@ -0,0 +1,121 @@
+/* Overrides of notebook CSS for static HTML export */
+.reveal {
+  font-size: 160%;
+}
+.reveal table {
+  font-size: var(--jp-ui-font-size1);
+}
+.reveal pre {
+  width: inherit;
+  padding: 0.4em;
+  margin: 0px;
+  font-family: monospace, sans-serif;
+  font-size: 80%;
+  box-shadow: 0px 0px 0px rgba(0, 0, 0, 0);
+}
+.reveal pre code {
+  padding: 0px;
+}
+.reveal section img {
+  border: 0px solid black;
+  box-shadow: 0 0 10px rgba(0, 0, 0, 0);
+}
+.reveal .slides {
+  text-align: left;
+}
+.reveal.fade {
+  opacity: 1;
+}
+.reveal .progress {
+  position: static;
+}
+
+div.jp-InputArea-editor {
+  padding: 0.06em;
+}
+
+div.code_cell {
+  background-color: transparent;
+}
+
+div.output_area pre {
+  font-family: monospace, sans-serif;
+  font-size: 80%;
+}
+
+div.jp-OutputPrompt {
+  /* 5px right shift to account for margin in parent container */
+  margin: 5px 5px 0 0;
+}
+
+.reveal div.highlight {
+  margin: 0;
+}
+
+.reveal div.highlight > pre {
+  margin: 0;
+  width: 100%;
+  font-size: var(--jp-code-font-size);
+}
+
+.reveal div.jp-OutputArea-output > pre {
+  margin: 0;
+  width: 90%;
+  font-size: var(--jp-code-font-size);
+  box-shadow: none;
+}
+
+main {
+  height: 100%;
+}
+
+/* Reveal navigation controls */
+
+.reveal .controls .navigate-left,
+.reveal .controls .navigate-left.enabled {
+  border-right-color: #727272;
+}
+.reveal .controls .navigate-left.enabled:hover,
+.reveal .controls .navigate-left.enabled.enabled:hover {
+  border-right-color: #dfdfdf;
+}
+.reveal .controls .navigate-right,
+.reveal .controls .navigate-right.enabled {
+  border-left-color: #727272;
+}
+.reveal .controls .navigate-right.enabled:hover,
+.reveal .controls .navigate-right.enabled.enabled:hover {
+  border-left-color: #dfdfdf;
+}
+.reveal .controls .navigate-up,
+.reveal .controls .navigate-up.enabled {
+  border-bottom-color: #727272;
+}
+.reveal .controls .navigate-up.enabled:hover,
+.reveal .controls .navigate-up.enabled.enabled:hover {
+  border-bottom-color: #dfdfdf;
+}
+.reveal .controls .navigate-down,
+.reveal .controls .navigate-down.enabled {
+  border-top-color: #727272;
+}
+.reveal .controls .navigate-down.enabled:hover,
+.reveal .controls .navigate-down.enabled.enabled:hover {
+  border-top-color: #dfdfdf;
+}
+.reveal .progress span {
+  background: #727272;
+}
+
+/* Scrollbars */
+
+::-webkit-scrollbar {
+  width: 6px;
+  height: 6px;
+}
+::-webkit-scrollbar * {
+  background: transparent;
+}
+::-webkit-scrollbar-thumb {
+  background: #727272 !important;
+}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/rst/conf.json

@@ -0,0 +1,6 @@
+{
+  "base_template": "base",
+  "mimetypes": {
+    "text/x-rst": true
+  }
+}

emissary-ml/llm-scripts/fine-tuning/llama3/venv/share/jupyter/nbconvert/templates/rst/index.rst.j2

@@ -0,0 +1,117 @@
+{%- extends 'display_priority.j2' -%}
+
+
+{% block in_prompt %}
+{% endblock in_prompt %}
+
+{% block output_prompt %}
+{% endblock output_prompt %}
+
+{% block input scoped%}
+{%- if cell.source.strip() -%}
+{{".. code:: "-}}
+{%- if 'magics_language' in cell.metadata  -%}
+    {{ cell.metadata.magics_language}}
+{%- elif 'pygments_lexer' in nb.metadata.get('language_info', {}) -%}
+    {{ nb.metadata.language_info.pygments_lexer }}
+{%- elif 'name' in nb.metadata.get('language_info', {}) -%}
+    {{ nb.metadata.language_info.name }}
+{%- endif %}
+
+{{ cell.source | indent}}
+{% endif -%}
+{% endblock input %}
+
+{% block error %}
+::
+
+{{ super() }}
+{% endblock error %}
+
+{% block traceback_line %}
+{{ line | indent | strip_ansi }}
+{% endblock traceback_line %}
+
+{% block execute_result %}
+{% block data_priority scoped %}
+{{ super() }}
+{% endblock %}
+{% endblock execute_result %}
+
+{% block stream %}
+.. parsed-literal::
+
+{{ output.text | indent }}
+{% endblock stream %}
+
+{% block data_native %}
+{{ output.data['text/x-rst'] }}
+{% endblock data_native %}
+
+{% block data_svg %}
+.. image:: {{ output.metadata.filenames['image/svg+xml'] | urlencode }}
+{% endblock data_svg %}
+
+{% block data_png %}
+.. image:: {{ output.metadata.filenames['image/png'] | urlencode }}
+{%- set width=output | get_metadata('width', 'image/png') -%}
+{%- if width is not none %}
+   :width: {{ width }}px
+{%- endif %}
+{%- set height=output | get_metadata('height', 'image/png') -%}
+{%- if height is not none %}
+   :height: {{ height }}px
+{%- endif %}
+{% endblock data_png %}
+
+{% block data_jpg %}
+.. image:: {{ output.metadata.filenames['image/jpeg'] | urlencode }}
+{%- set width=output | get_metadata('width', 'image/jpeg') -%}
+{%- if width is not none %}
+   :width: {{ width }}px
+{%- endif %}
+{%- set height=output | get_metadata('height', 'image/jpeg') -%}
+{%- if height is not none %}
+   :height: {{ height }}px
+{%- endif %}
+{% endblock data_jpg %}
+
+{% block data_markdown %}
+{{ output.data['text/markdown'] | convert_pandoc("markdown", "rst") }}
+{% endblock data_markdown %}
+
+{% block data_latex %}
+.. math::
+
+{{ output.data['text/latex'] | strip_dollars | indent }}
+{% endblock data_latex %}
+
+{% block data_text scoped %}
+.. parsed-literal::
+
+{{ output.data['text/plain'] | indent }}
+{% endblock data_text %}
+
+{% block data_html scoped %}
+.. raw:: html
+
+{{ output.data['text/html'] | indent }}
+{% endblock data_html %}
+
+{% block markdowncell scoped %}
+{{ cell.source | convert_pandoc("markdown", "rst") }}
+{% endblock markdowncell %}
+
+{%- block rawcell scoped -%}
+{%- if cell.metadata.get('raw_mimetype', '').lower() in resources.get('raw_mimetypes', ['']) %}
+{{cell.source}}
+{% endif -%}
+{%- endblock rawcell -%}
+
+{% block headingcell scoped %}
+{{ ("#" * cell.level + cell.source) | replace('\n', ' ') | convert_pandoc("markdown", "rst") }}
+{% endblock headingcell %}
+
+{% block unknowncell scoped %}
+unknown type  {{cell.type}}
+{% endblock unknowncell %}

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/debug.log

@@ -0,0 +1,34 @@
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Configure stats pid to 96305
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Loading settings from /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/settings
+2025-08-11 23:43:33,509 WARNING MainThread:96305 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_KEY
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'llm-scripts/fine-tuning/llama3/train.py', 'program_abspath': '/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/train.py', 'program': '/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/train.py'}
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_init.py:_log_setup():527] Logging user logs to /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_234333-63gx7jwn/logs/debug.log
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_init.py:_log_setup():528] Logging internal logs to /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_234333-63gx7jwn/logs/debug-internal.log
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_init.py:init():567] calling init triggers
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_init.py:init():574] wandb.init called with sweep_config: {}
+config: {}
+2025-08-11 23:43:33,509 INFO    MainThread:96305 [wandb_init.py:init():617] starting backend
+2025-08-11 23:43:33,510 INFO    MainThread:96305 [wandb_init.py:init():621] setting up manager
+2025-08-11 23:43:33,512 INFO    MainThread:96305 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-08-11 23:43:33,513 INFO    MainThread:96305 [wandb_init.py:init():629] backend started and connected
+2025-08-11 23:43:33,521 INFO    MainThread:96305 [wandb_init.py:init():721] updated telemetry
+2025-08-11 23:43:33,526 INFO    MainThread:96305 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout
+2025-08-11 23:43:33,828 INFO    MainThread:96305 [wandb_run.py:_on_init():2344] communicating current version
+2025-08-11 23:43:33,868 INFO    MainThread:96305 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.21.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-08-11 23:43:33,868 INFO    MainThread:96305 [wandb_init.py:init():805] starting run threads in backend
+2025-08-11 23:43:34,670 INFO    MainThread:96305 [wandb_run.py:_console_start():2323] atexit reg
+2025-08-11 23:43:34,670 INFO    MainThread:96305 [wandb_run.py:_redirect():2178] redirect: wrap_raw
+2025-08-11 23:43:34,670 INFO    MainThread:96305 [wandb_run.py:_redirect():2243] Wrapping output streams.
+2025-08-11 23:43:34,671 INFO    MainThread:96305 [wandb_run.py:_redirect():2268] Redirects installed.
+2025-08-11 23:43:34,671 INFO    MainThread:96305 [wandb_init.py:init():848] run started, returning control to user process
+2025-08-11 23:47:14,511 INFO    MainThread:96305 [wandb_run.py:_config_callback():1347] config_cb None None {'peft_config': {'default': {'task_type': 'CAUSAL_LM', 'peft_type': <PeftType.LORA: 'LORA'>, 'auto_mapping': None, 'base_model_name_or_path': 'emissary-ai/Python-Tab-Completion-CodeLlama-70b', 'revision': None, 'inference_mode': False, 'r': 16, 'target_modules': {'v_proj', 'up_proj', 'gate_proj', 'q_proj', 'k_proj', 'down_proj', 'o_proj'}, 'exclude_modules': None, 'lora_alpha': 64, 'lora_dropout': 0.05, 'fan_in_fan_out': False, 'bias': 'none', 'use_rslora': False, 'modules_to_save': None, 'init_lora_weights': True, 'layers_to_transform': None, 'layers_pattern': None, 'rank_pattern': {}, 'alpha_pattern': {}, 'megatron_config': None, 'megatron_core': 'megatron.core', 'trainable_token_indices': None, 'loftq_config': {}, 'eva_config': None, 'corda_config': None, 'use_dora': False, 'use_qalora': False, 'qalora_group_size': 16, 'layer_replication': None, 'runtime_config': {'ephemeral_gpu_offload': False}, 'lora_bias': False, 'target_parameters': None}}, 'vocab_size': 32016, 'max_position_embeddings': 131072, 'hidden_size': 8192, 'intermediate_size': 28672, 'num_hidden_layers': 80, 'num_attention_heads': 64, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 10000, 'rope_scaling': None, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'torchscript': False, 'torch_dtype': 'float16', 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'task_specific_params': None, 'problem_type': None, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 2, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, '_name_or_path': 'emissary-ai/Python-Tab-Completion-CodeLlama-70b', 'transformers_version': '4.55.0', 'model_type': 'llama', 'tf_legacy_loss': False, 'use_bfloat16': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': True, '_load_in_4bit': False, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'fp4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float32', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': False, 'load_in_8bit': True}, 'output_attentions': False, 'output_dir': 'checkpoints', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 16, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0002, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 0, 'max_steps': 0, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'checkpoints/runs/Aug11_23-47-13_shadecloud', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 99999, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 3407, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_8bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'model_init_kwargs': None, 'use_liger': False, 'dataset_text_field': 'text', 'dataset_kwargs': None, 'dataset_num_proc': None, 'max_seq_length': 1024, 'packing': False, 'eval_packing': None, 'dataset_batch_size': None, 'num_of_sequences': None, 'chars_per_token': '<CHARS_PER_TOKEN>'}
+2025-08-11 23:47:14,520 INFO    MainThread:96305 [wandb_config.py:__setitem__():151] config set model/num_parameters = 69184004096 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f75842f5780>>
+2025-08-11 23:47:14,520 INFO    MainThread:96305 [wandb_run.py:_config_callback():1347] config_cb model/num_parameters 69184004096 None
+2025-08-11 23:47:55,627 WARNING MsgRouterThr:96305 [router.py:message_loop():77] message_loop has been closed

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_231120-bt4spvmx/files/wandb-metadata.json

@@ -0,0 +1,741 @@
+{
+    "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2025-08-11T23:11:21.233602",
+    "startedAt": "2025-08-11T23:11:20.295305",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--dataset_path",
+        "/home/shadeform/tr-5ekuRmvnwgDznEKHAQcBL2-train.jsonl",
+        "--dataset_id",
+        "ds-9eYnuJhismAkC4AqzDXuVo",
+        "--test_dataset_path",
+        "/home/shadeform/tr-5ekuRmvnwgDznEKHAQcBL2-test.jsonl",
+        "--test_data_id",
+        "ds-MLF2XiyMxXD4CPsrLwKYAY",
+        "--service_id",
+        "ms-DbLTHy7dKWzyFzjvLDENkY",
+        "--training_id",
+        "tr-Mwx5sYhG3PnojUx9dNq3Bn",
+        "--user_id",
+        "user_2dNEee5lJORqWKAhGHurdhTLvEQ",
+        "--task_type",
+        "text-generation",
+        "--model_name",
+        "Llama3.1-8B-Instruct",
+        "--base_model_link",
+        "emissary-ai/Python-Tab-Completion-CodeLlama-70b",
+        "--is_external",
+        "false",
+        "--training_technique",
+        "SFT"
+    ],
+    "state": "running",
+    "program": "/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/train.py",
+    "codePathLocal": "train.py",
+    "codePath": "llm-scripts/fine-tuning/llama3/train.py",
+    "git": {
+        "remote": "https://coloryourlife:@github.com/Emissary-Tech/emissary-ml.git",
+        "commit": "0143ec41bec5a001ce9b65ac366aa9ddd6cdc966"
+    },
+    "email": null,
+    "root": "/home/shadeform/emissary-ml",
+    "host": "shadecloud",
+    "username": "root",
+    "executable": "/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/venv/bin/python",
+    "cpu_count": 64,
+    "cpu_count_logical": 128,
+    "cpu_freq": {
+        "current": 2236.7202578124998,
+        "min": 1500.0,
+        "max": 3250.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3800.218,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 2300.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 3250.0,
+            "min": 1500.0,
+            "max": 3250.0
+        },
+        {
+            "current": 1500.0,
+            "min": 1500.0,
+            "max": 3250.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 3519.1512603759766,
+            "used": 706.6665191650391
+        }
+    },
+    "gpu": "NVIDIA H100 PCIe",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 PCIe",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 1505.468677520752
+    }
+}

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_231120-bt4spvmx/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train_runtime": 0.0134, "train_samples_per_second": 0.0, "train_steps_per_second": 0.0, "total_flos": 0, "train_loss": 0.0, "train/epoch": 0, "train/global_step": 0, "_timestamp": 1754953912.0067096, "_runtime": 31.708322525024414, "_step": 0, "_wandb": {"runtime": 52}}

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_231120-bt4spvmx/logs/debug.log

@@ -0,0 +1,34 @@
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Current SDK version is 0.16.5
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Configure stats pid to 94538
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Loading settings from /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/settings
+2025-08-11 23:11:20,296 WARNING MainThread:94538 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_KEY
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'llm-scripts/fine-tuning/llama3/train.py', 'program_abspath': '/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/train.py', 'program': '/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/train.py'}
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_init.py:_log_setup():527] Logging user logs to /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_231120-bt4spvmx/logs/debug.log
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_init.py:_log_setup():528] Logging internal logs to /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_231120-bt4spvmx/logs/debug-internal.log
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_init.py:init():567] calling init triggers
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_init.py:init():574] wandb.init called with sweep_config: {}
+config: {}
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_init.py:init():617] starting backend
+2025-08-11 23:11:20,296 INFO    MainThread:94538 [wandb_init.py:init():621] setting up manager
+2025-08-11 23:11:20,297 INFO    MainThread:94538 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-08-11 23:11:20,298 INFO    MainThread:94538 [wandb_init.py:init():629] backend started and connected
+2025-08-11 23:11:20,299 INFO    MainThread:94538 [wandb_init.py:init():721] updated telemetry
+2025-08-11 23:11:20,302 INFO    MainThread:94538 [wandb_init.py:init():754] communicating run to backend with 90.0 second timeout
+2025-08-11 23:11:20,534 INFO    MainThread:94538 [wandb_run.py:_on_init():2344] communicating current version
+2025-08-11 23:11:20,561 INFO    MainThread:94538 [wandb_run.py:_on_init():2353] got version response upgrade_message: "wandb version 0.21.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-08-11 23:11:20,561 INFO    MainThread:94538 [wandb_init.py:init():805] starting run threads in backend
+2025-08-11 23:11:21,360 INFO    MainThread:94538 [wandb_run.py:_console_start():2323] atexit reg
+2025-08-11 23:11:21,360 INFO    MainThread:94538 [wandb_run.py:_redirect():2178] redirect: wrap_raw
+2025-08-11 23:11:21,360 INFO    MainThread:94538 [wandb_run.py:_redirect():2243] Wrapping output streams.
+2025-08-11 23:11:21,360 INFO    MainThread:94538 [wandb_run.py:_redirect():2268] Redirects installed.
+2025-08-11 23:11:21,360 INFO    MainThread:94538 [wandb_init.py:init():848] run started, returning control to user process
+2025-08-11 23:11:51,998 INFO    MainThread:94538 [wandb_run.py:_config_callback():1347] config_cb None None {'peft_config': {'default': {'task_type': 'CAUSAL_LM', 'peft_type': <PeftType.LORA: 'LORA'>, 'auto_mapping': None, 'base_model_name_or_path': 'meta-llama/Llama-3.1-8B-Instruct', 'revision': None, 'inference_mode': False, 'r': 16, 'target_modules': {'k_proj', 'v_proj', 'gate_proj', 'q_proj', 'down_proj', 'o_proj', 'up_proj'}, 'exclude_modules': None, 'lora_alpha': 64, 'lora_dropout': 0.05, 'fan_in_fan_out': False, 'bias': 'none', 'use_rslora': False, 'modules_to_save': None, 'init_lora_weights': True, 'layers_to_transform': None, 'layers_pattern': None, 'rank_pattern': {}, 'alpha_pattern': {}, 'megatron_config': None, 'megatron_core': 'megatron.core', 'trainable_token_indices': None, 'loftq_config': {}, 'eva_config': None, 'corda_config': None, 'use_dora': False, 'use_qalora': False, 'qalora_group_size': 16, 'layer_replication': None, 'runtime_config': {'ephemeral_gpu_offload': False}, 'lora_bias': False, 'target_parameters': None}}, 'vocab_size': 128257, 'max_position_embeddings': 131072, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 32, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 128, 'return_dict': True, 'output_hidden_states': False, 'torchscript': False, 'torch_dtype': 'float16', 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'task_specific_params': None, 'problem_type': None, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': 128256, 'eos_token_id': [128001, 128008, 128009], 'sep_token_id': None, 'decoder_start_token_id': None, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, '_name_or_path': 'meta-llama/Llama-3.1-8B-Instruct', 'transformers_version': '4.55.0', 'model_type': 'llama', 'tf_legacy_loss': False, 'use_bfloat16': False, 'quantization_config': {'quant_method': 'QuantizationMethod.BITS_AND_BYTES', '_load_in_8bit': True, '_load_in_4bit': False, 'llm_int8_threshold': 6.0, 'llm_int8_skip_modules': None, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'bnb_4bit_quant_type': 'fp4', 'bnb_4bit_use_double_quant': False, 'bnb_4bit_compute_dtype': 'float32', 'bnb_4bit_quant_storage': 'uint8', 'load_in_4bit': False, 'load_in_8bit': True}, 'output_attentions': False, 'output_dir': 'checkpoints', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 16, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0002, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 0.3, 'num_train_epochs': 0, 'max_steps': 0, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'checkpoints/runs/Aug11_23-11-50_shadecloud', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 99999, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 3407, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_8bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': True, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'model_init_kwargs': None, 'use_liger': False, 'dataset_text_field': 'text', 'dataset_kwargs': None, 'dataset_num_proc': None, 'max_seq_length': 1024, 'packing': False, 'eval_packing': None, 'dataset_batch_size': None, 'num_of_sequences': None, 'chars_per_token': '<CHARS_PER_TOKEN>'}
+2025-08-11 23:11:52,003 INFO    MainThread:94538 [wandb_config.py:__setitem__():151] config set model/num_parameters = 8072212480 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f62fdeb0610>>
+2025-08-11 23:11:52,003 INFO    MainThread:94538 [wandb_run.py:_config_callback():1347] config_cb model/num_parameters 8072212480 None
+2025-08-11 23:12:20,167 WARNING MsgRouterThr:94538 [router.py:message_loop():77] message_loop has been closed

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/config.yaml

@@ -0,0 +1,46 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.5
+    framework: huggingface
+    huggingface_version: 4.55.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1754955408.0
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      - 98
+      2:
+      - 1
+      - 5
+      - 11
+      - 41
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      - 98
+      3:
+      - 15
+      - 23
+      4: 3.10.12
+      5: 0.16.5
+      6: 4.55.0
+      8:
+      - 5
+      13: linux-x86_64

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 1}}

emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/logs/debug-internal.log

@@ -0,0 +1,184 @@
+2025-08-11 23:36:48,180 INFO    StreamThr :95897 [internal.py:wandb_internal():86] W&B internal server running at pid: 95897, started at: 2025-08-11 23:36:48.179572
+2025-08-11 23:36:48,181 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: status
+2025-08-11 23:36:48,182 INFO    WriterThread:95897 [datastore.py:open_for_write():87] open: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/run-eg3v9sv7.wandb
+2025-08-11 23:36:48,184 DEBUG   SenderThread:95897 [sender.py:send():379] send: header
+2025-08-11 23:36:48,185 DEBUG   SenderThread:95897 [sender.py:send():379] send: run
+2025-08-11 23:36:48,426 INFO    SenderThread:95897 [dir_watcher.py:__init__():211] watching files in: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files
+2025-08-11 23:36:48,426 INFO    SenderThread:95897 [sender.py:_start_run_threads():1124] run started: eg3v9sv7 with start time 1754955408.179257
+2025-08-11 23:36:48,431 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: check_version
+2025-08-11 23:36:48,431 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: check_version
+2025-08-11 23:36:48,509 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: run_start
+2025-08-11 23:36:49,081 DEBUG   HandlerThread:95897 [system_info.py:__init__():26] System info init
+2025-08-11 23:36:49,081 DEBUG   HandlerThread:95897 [system_info.py:__init__():41] System info init done
+2025-08-11 23:36:49,081 INFO    HandlerThread:95897 [system_monitor.py:start():194] Starting system monitor
+2025-08-11 23:36:49,082 INFO    SystemMonitor:95897 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-08-11 23:36:49,082 INFO    HandlerThread:95897 [system_monitor.py:probe():214] Collecting system info
+2025-08-11 23:36:49,082 INFO    SystemMonitor:95897 [interfaces.py:start():190] Started cpu monitoring
+2025-08-11 23:36:49,083 INFO    SystemMonitor:95897 [interfaces.py:start():190] Started disk monitoring
+2025-08-11 23:36:49,084 INFO    SystemMonitor:95897 [interfaces.py:start():190] Started gpu monitoring
+2025-08-11 23:36:49,084 INFO    SystemMonitor:95897 [interfaces.py:start():190] Started memory monitoring
+2025-08-11 23:36:49,085 INFO    SystemMonitor:95897 [interfaces.py:start():190] Started network monitoring
+2025-08-11 23:36:49,142 DEBUG   HandlerThread:95897 [system_info.py:probe():150] Probing system
+2025-08-11 23:36:49,144 DEBUG   HandlerThread:95897 [system_info.py:_probe_git():135] Probing git
+2025-08-11 23:36:49,149 DEBUG   HandlerThread:95897 [system_info.py:_probe_git():143] Probing git done
+2025-08-11 23:36:49,149 DEBUG   HandlerThread:95897 [system_info.py:probe():198] Probing system done
+2025-08-11 23:36:49,149 DEBUG   HandlerThread:95897 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-151-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2025-08-11T23:36:49.142476', 'startedAt': '2025-08-11T23:36:48.176106', 'docker': None, 'cuda': None, 'args': ('--dataset_path', '/home/shadeform/tr-5ekuRmvnwgDznEKHAQcBL2-train.jsonl', '--dataset_id', 'ds-9eYnuJhismAkC4AqzDXuVo', '--test_dataset_path', '/home/shadeform/tr-5ekuRmvnwgDznEKHAQcBL2-test.jsonl', '--test_data_id', 'ds-MLF2XiyMxXD4CPsrLwKYAY', '--service_id', 'ms-DbLTHy7dKWzyFzjvLDENkY', '--training_id', 'tr-Mwx5sYhG3PnojUx9dNq3Bn', '--user_id', 'user_2dNEee5lJORqWKAhGHurdhTLvEQ', '--task_type', 'text-generation', '--model_name', 'python-tab-completion-codellama-70b', '--base_model_link', 'emissary-ai/Python-Tab-Completion-CodeLlama-70b', '--is_external', 'false', '--training_technique', 'SFT'), 'state': 'running', 'program': '/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/train.py', 'codePathLocal': 'train.py', 'codePath': 'llm-scripts/fine-tuning/llama3/train.py', 'git': {'remote': 'https://coloryourlife:@github.com/Emissary-Tech/emissary-ml.git', 'commit': '0143ec41bec5a001ce9b65ac366aa9ddd6cdc966'}, 'email': None, 'root': '/home/shadeform/emissary-ml', 'host': 'shadecloud', 'username': 'root', 'executable': '/home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/venv/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 2024.2199609375, 'min': 1500.0, 'max': 3250.0}, 'cpu_freq_per_core': [{'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3800.214, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 1500.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 2300.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}, {'current': 3250.0, 'min': 1500.0, 'max': 3250.0}], 'disk': {'/': {'total': 3519.1512603759766, 'used': 729.7996559143066}}, 'gpu': 'NVIDIA H100 PCIe', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 PCIe', 'memory_total': 85520809984}], 'memory': {'total': 1505.468677520752}}
+2025-08-11 23:36:49,149 INFO    HandlerThread:95897 [system_monitor.py:probe():224] Finished collecting system info
+2025-08-11 23:36:49,149 INFO    HandlerThread:95897 [system_monitor.py:probe():227] Publishing system info
+2025-08-11 23:36:49,150 INFO    HandlerThread:95897 [system_monitor.py:probe():229] Finished publishing system info
+2025-08-11 23:36:49,152 DEBUG   SenderThread:95897 [sender.py:send():379] send: files
+2025-08-11 23:36:49,152 INFO    SenderThread:95897 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now
+2025-08-11 23:36:49,292 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: python_packages
+2025-08-11 23:36:49,293 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: python_packages
+2025-08-11 23:36:49,293 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: stop_status
+2025-08-11 23:36:49,294 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: stop_status
+2025-08-11 23:36:49,296 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: internal_messages
+2025-08-11 23:36:49,410 DEBUG   SenderThread:95897 [sender.py:send():379] send: telemetry
+2025-08-11 23:36:49,424 INFO    wandb-upload_0:95897 [upload_job.py:push():131] Uploaded file /tmp/tmpbvz6x99qwandb/p2ok8soc-wandb-metadata.json
+2025-08-11 23:36:49,427 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_created():271] file/dir created: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/output.log
+2025-08-11 23:36:49,427 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_created():271] file/dir created: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/requirements.txt
+2025-08-11 23:36:49,427 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_created():271] file/dir created: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-metadata.json
+2025-08-11 23:36:50,267 DEBUG   SenderThread:95897 [sender.py:send():379] send: exit
+2025-08-11 23:36:50,267 INFO    SenderThread:95897 [sender.py:send_exit():586] handling exit code: 0
+2025-08-11 23:36:50,267 INFO    SenderThread:95897 [sender.py:send_exit():588] handling runtime: 1
+2025-08-11 23:36:50,267 INFO    SenderThread:95897 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2025-08-11 23:36:50,267 INFO    SenderThread:95897 [sender.py:send_exit():594] send defer
+2025-08-11 23:36:50,267 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:50,267 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 0
+2025-08-11 23:36:50,268 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:50,268 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-08-11 23:36:50,268 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 1
+2025-08-11 23:36:50,268 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:50,268 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 1
+2025-08-11 23:36:50,268 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:50,268 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-08-11 23:36:50,268 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 2
+2025-08-11 23:36:50,268 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:50,268 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 2
+2025-08-11 23:36:50,268 INFO    HandlerThread:95897 [system_monitor.py:finish():203] Stopping system monitor
+2025-08-11 23:36:50,268 DEBUG   SystemMonitor:95897 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-08-11 23:36:50,269 DEBUG   SystemMonitor:95897 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-08-11 23:36:50,269 DEBUG   SystemMonitor:95897 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-08-11 23:36:50,269 INFO    HandlerThread:95897 [interfaces.py:finish():202] Joined cpu monitor
+2025-08-11 23:36:50,270 INFO    HandlerThread:95897 [interfaces.py:finish():202] Joined disk monitor
+2025-08-11 23:36:50,427 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_created():271] file/dir created: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-summary.json
+2025-08-11 23:36:51,428 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/output.log
+2025-08-11 23:36:51,457 INFO    HandlerThread:95897 [interfaces.py:finish():202] Joined gpu monitor
+2025-08-11 23:36:51,457 INFO    HandlerThread:95897 [interfaces.py:finish():202] Joined memory monitor
+2025-08-11 23:36:51,457 INFO    HandlerThread:95897 [interfaces.py:finish():202] Joined network monitor
+2025-08-11 23:36:51,458 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: poll_exit
+2025-08-11 23:36:51,459 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:51,459 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-08-11 23:36:51,459 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 3
+2025-08-11 23:36:51,459 DEBUG   SenderThread:95897 [sender.py:send():379] send: stats
+2025-08-11 23:36:51,459 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: poll_exit
+2025-08-11 23:36:51,460 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:51,460 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 3
+2025-08-11 23:36:51,460 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:51,461 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-08-11 23:36:51,461 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 4
+2025-08-11 23:36:51,461 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:51,461 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 4
+2025-08-11 23:36:51,461 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:51,461 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-08-11 23:36:51,461 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 5
+2025-08-11 23:36:51,462 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:51,462 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 5
+2025-08-11 23:36:51,462 DEBUG   SenderThread:95897 [sender.py:send():379] send: summary
+2025-08-11 23:36:51,463 INFO    SenderThread:95897 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2025-08-11 23:36:51,463 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:51,463 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-08-11 23:36:51,463 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 6
+2025-08-11 23:36:51,463 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:51,464 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 6
+2025-08-11 23:36:51,464 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:51,464 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-08-11 23:36:51,469 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: status_report
+2025-08-11 23:36:51,540 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 7
+2025-08-11 23:36:51,541 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:51,541 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 7
+2025-08-11 23:36:51,541 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:51,541 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-08-11 23:36:52,268 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: poll_exit
+2025-08-11 23:36:52,428 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/output.log
+2025-08-11 23:36:52,429 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/config.yaml
+2025-08-11 23:36:52,429 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-summary.json
+2025-08-11 23:36:54,031 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 8
+2025-08-11 23:36:54,031 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: poll_exit
+2025-08-11 23:36:54,032 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,032 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 8
+2025-08-11 23:36:54,033 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,033 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-08-11 23:36:54,033 INFO    SenderThread:95897 [job_builder.py:build():318] Attempting to build job artifact
+2025-08-11 23:36:54,033 INFO    SenderThread:95897 [job_builder.py:_get_source_type():455] is repo sourced job
+2025-08-11 23:36:54,060 INFO    SenderThread:95897 [job_builder.py:build():431] adding wandb-job metadata file
+2025-08-11 23:36:54,063 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 9
+2025-08-11 23:36:54,064 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,064 DEBUG   SenderThread:95897 [sender.py:send():379] send: artifact
+2025-08-11 23:36:54,064 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 9
+2025-08-11 23:36:54,271 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: poll_exit
+2025-08-11 23:36:54,350 INFO    SenderThread:95897 [sender.py:send_artifact():1468] sent artifact job-https___coloryourlife__github.com_Emissary-Tech_emissary-ml.git_llm-scripts_fine-tuning_llama3_train.py - {'id': 'QXJ0aWZhY3Q6MTk0ODg5MzA0Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjcxMTAwMjM0OQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTk0ODg5MzA0Nw==', 'versionIndex': 18}}}
+2025-08-11 23:36:54,350 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,350 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-08-11 23:36:54,350 INFO    SenderThread:95897 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-08-11 23:36:54,430 INFO    Thread-12 :95897 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/output.log
+2025-08-11 23:36:54,431 INFO    SenderThread:95897 [dir_watcher.py:finish():388] scan: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files
+2025-08-11 23:36:54,431 INFO    SenderThread:95897 [dir_watcher.py:finish():402] scan save: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/output.log output.log
+2025-08-11 23:36:54,431 INFO    SenderThread:95897 [dir_watcher.py:finish():402] scan save: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-metadata.json wandb-metadata.json
+2025-08-11 23:36:54,431 INFO    SenderThread:95897 [dir_watcher.py:finish():402] scan save: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/config.yaml config.yaml
+2025-08-11 23:36:54,432 INFO    SenderThread:95897 [dir_watcher.py:finish():402] scan save: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/requirements.txt requirements.txt
+2025-08-11 23:36:54,434 INFO    SenderThread:95897 [dir_watcher.py:finish():402] scan save: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-summary.json wandb-summary.json
+2025-08-11 23:36:54,438 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 10
+2025-08-11 23:36:54,438 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: poll_exit
+2025-08-11 23:36:54,441 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,441 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 10
+2025-08-11 23:36:54,444 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,444 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-08-11 23:36:54,444 INFO    SenderThread:95897 [file_pusher.py:finish():172] shutting down file pusher
+2025-08-11 23:36:54,622 INFO    wandb-upload_0:95897 [upload_job.py:push():131] Uploaded file /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/output.log
+2025-08-11 23:36:54,640 INFO    wandb-upload_1:95897 [upload_job.py:push():131] Uploaded file /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/config.yaml
+2025-08-11 23:36:54,650 INFO    wandb-upload_2:95897 [upload_job.py:push():131] Uploaded file /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/requirements.txt
+2025-08-11 23:36:54,667 INFO    wandb-upload_3:95897 [upload_job.py:push():131] Uploaded file /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/files/wandb-summary.json
+2025-08-11 23:36:54,867 INFO    Thread-11 (_thread_body):95897 [sender.py:transition_state():614] send defer: 11
+2025-08-11 23:36:54,868 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,868 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 11
+2025-08-11 23:36:54,869 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,869 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-08-11 23:36:54,869 INFO    SenderThread:95897 [file_pusher.py:join():178] waiting for file pusher
+2025-08-11 23:36:54,869 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 12
+2025-08-11 23:36:54,870 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,870 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 12
+2025-08-11 23:36:54,870 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,870 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-08-11 23:36:54,870 INFO    SenderThread:95897 [file_stream.py:finish():614] file stream finish called
+2025-08-11 23:36:54,948 INFO    SenderThread:95897 [file_stream.py:finish():618] file stream finish is done
+2025-08-11 23:36:54,948 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 13
+2025-08-11 23:36:54,948 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,948 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 13
+2025-08-11 23:36:54,949 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,949 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-08-11 23:36:54,949 INFO    SenderThread:95897 [sender.py:transition_state():614] send defer: 14
+2025-08-11 23:36:54,949 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: defer
+2025-08-11 23:36:54,950 INFO    HandlerThread:95897 [handler.py:handle_request_defer():172] handle defer: 14
+2025-08-11 23:36:54,950 DEBUG   SenderThread:95897 [sender.py:send():379] send: final
+2025-08-11 23:36:54,950 DEBUG   SenderThread:95897 [sender.py:send():379] send: footer
+2025-08-11 23:36:54,950 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: defer
+2025-08-11 23:36:54,950 INFO    SenderThread:95897 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-08-11 23:36:54,951 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: poll_exit
+2025-08-11 23:36:54,951 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: poll_exit
+2025-08-11 23:36:54,952 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: poll_exit
+2025-08-11 23:36:54,952 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: poll_exit
+2025-08-11 23:36:54,953 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: server_info
+2025-08-11 23:36:54,953 DEBUG   SenderThread:95897 [sender.py:send_request():406] send_request: server_info
+2025-08-11 23:36:54,956 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: get_summary
+2025-08-11 23:36:54,956 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: sampled_history
+2025-08-11 23:36:54,957 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: internal_messages
+2025-08-11 23:36:55,000 INFO    MainThread:95897 [wandb_run.py:_footer_history_summary_info():3920] rendering history
+2025-08-11 23:36:55,000 INFO    MainThread:95897 [wandb_run.py:_footer_history_summary_info():3952] rendering summary
+2025-08-11 23:36:55,000 INFO    MainThread:95897 [wandb_run.py:_footer_sync_info():3879] logging synced files
+2025-08-11 23:36:55,001 DEBUG   HandlerThread:95897 [handler.py:handle_request():146] handle_request: shutdown
+2025-08-11 23:36:55,001 INFO    HandlerThread:95897 [handler.py:finish():866] shutting down handler
+2025-08-11 23:36:55,953 INFO    WriterThread:95897 [datastore.py:close():296] close: /home/shadeform/emissary-ml/llm-scripts/fine-tuning/llama3/wandb/run-20250811_233648-eg3v9sv7/run-eg3v9sv7.wandb
+2025-08-11 23:36:56,000 INFO    SenderThread:95897 [sender.py:finish():1546] shutting down sender
+2025-08-11 23:36:56,000 INFO    SenderThread:95897 [file_pusher.py:finish():172] shutting down file pusher
+2025-08-11 23:36:56,000 INFO    SenderThread:95897 [file_pusher.py:join():178] waiting for file pusher