File size: 2,903 Bytes
4cb6700
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
"""
ROTATING STDOUT - Dual logs: 5000 lines (main) + 2500 lines (compact)
"""
import sys
import os
from datetime import datetime

MAX_LINES = 5000
MAX_LINES_COMPACT = 2500
LOG_FILE = os.environ.get('TRAINING_LOG', '/root/training.log')
LOG_FILE_COMPACT = LOG_FILE.replace('.log', '_compact.log')
ARCHIVE_DIR = LOG_FILE.replace('.log', '_archives')

class RotatingStdout:
    def __init__(self, max_lines=MAX_LINES, max_compact=MAX_LINES_COMPACT):
        self.max_lines = max_lines
        self.max_compact = max_compact
        self.line_count = 0
        self.line_count_compact = 0
        self.original_stdout = sys.stdout
        self.rotating = False
        os.makedirs(ARCHIVE_DIR, exist_ok=True)
        self.log_file = open(LOG_FILE, 'a', buffering=1)
        self.log_compact = open(LOG_FILE_COMPACT, 'a', buffering=1)
    
    def _rotate_main(self):
        self.log_file.close()
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        archive_path = os.path.join(ARCHIVE_DIR, f'archive_{timestamp}.log')
        with open(LOG_FILE, 'r') as f:
            lines = f.readlines()
        keep = min(self.max_lines // 2, len(lines))
        with open(archive_path, 'w') as f:
            f.writelines(lines[:-keep] if keep else lines)
        with open(LOG_FILE, 'w') as f:
            f.writelines(lines[-keep:] if keep else [])
        self.line_count = keep
        self.log_file = open(LOG_FILE, 'a', buffering=1)
    
    def _rotate_compact(self):
        self.log_compact.close()
        with open(LOG_FILE_COMPACT, 'r') as f:
            lines = f.readlines()
        keep = min(self.max_compact // 2, len(lines))
        with open(LOG_FILE_COMPACT, 'w') as f:
            f.writelines(lines[-keep:] if keep else [])
        self.line_count_compact = keep
        self.log_compact = open(LOG_FILE_COMPACT, 'a', buffering=1)
    
    def _rotate_if_needed(self):
        if self.rotating:
            return
        self.rotating = True
        if self.line_count > self.max_lines:
            self._rotate_main()
        if self.line_count_compact > self.max_compact:
            self._rotate_compact()
        self.rotating = False
    
    def write(self, text):
        self.original_stdout.write(text)
        if not self.rotating:
            self.log_file.write(text)
            self.log_compact.write(text)
            newlines = text.count('\n')
            self.line_count += newlines
            self.line_count_compact += newlines
            self._rotate_if_needed()
    
    def flush(self):
        self.original_stdout.flush()
        if self.log_file:
            self.log_file.flush()
        if self.log_compact:
            self.log_compact.flush()

def install_rotating_log():
    rotating = RotatingStdout()
    sys.stdout = rotating
    sys.stderr = rotating
    print(f"[rotating_log] Active. Main={MAX_LINES}, Compact={MAX_LINES_COMPACT} lines.")