changs

README.md

@@ -8,3 +8,6 @@ pinned: false
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+

app.py

@@ -1,59 +1,99 @@
 
-import time
+
+
 import os
+import asyncio
+import tempfile
+from logbert_rca_pipeline_api import detect_anomalies_and_explain
 import redis
+import boto3
+from botocore.exceptions import ClientError
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import uvicorn
 
-import asyncio
-from sql import insert_rca_result, connect_to_database, disconnect_from_database
 
-# Directories
-UPLOAD_DIR = "/data/uploads"
-PROCESSED_DIR = "/data/processed"
-os.makedirs(PROCESSED_DIR, exist_ok=True)
+# FastAPI app
+app = FastAPI()
+
+# Initialize Redis client (adjust host/port/db as needed)
+redis_client = redis.Redis(host='localhost', port=6379, db=0)
+
+# Define the Redis queue name
+REDIS_QUEUE = "log_queue"
 
-# Redis configuration (must be set in environment variables)
-REDIS_HOST = os.environ["REDIS_HOST"]
-REDIS_PORT = int(os.environ["REDIS_PORT"])
-REDIS_QUEUE = os.environ["REDIS_QUEUE"]
-READY_FOR_RCA_QUEUE = os.environ.get("READY_FOR_RCA_QUEUE", "logbert_ready_for_rca")
+# Request model
+class LogRequest(BaseModel):
+    filename: str
 
-# Initialize Redis client
-redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
 
-def process_log(file_path):
-    # Dummy anomaly detection logic; replace with your real pipeline
-    # For example, call your detect_anomalies_and_explain here
-    return {"filename": os.path.basename(file_path), "anomaly": True, "details": "Example anomaly detected."}
+async def process_log(filename, file_content):
+    # Save file_content to a temporary file and run RCA pipeline
+    with tempfile.NamedTemporaryFile(delete=False, mode="wb", suffix=".log") as tmp:
+        tmp.write(file_content)
+        tmp_path = tmp.name
+    loop = asyncio.get_event_loop()
 
+    def _run_pipeline():
+        return detect_anomalies_and_explain(tmp_path)
+    results = await loop.run_in_executor(None, _run_pipeline)
+    os.unlink(tmp_path)
+    if results and len(results) > 0:
+        return results[0]
+    else:
+        return {"filename": filename, "anomaly": False, "details": "No anomaly detected."}
 
 
-def save_rca_to_db(rca_result):
-    # Store anomaly result in rca_results table using sql.py
-    async def _save():
-        await connect_to_database()
-        await insert_rca_result(rca_result)
-        await disconnect_from_database()
-    asyncio.run(_save())
+S3_BUCKET = "your-s3-bucket-name"
 
-def main():
+
+async def get_file_from_s3(filename):
+    loop = asyncio.get_event_loop()
+
+    def _download():
+        s3 = boto3.client("s3")
+        try:
+            response = s3.get_object(Bucket=S3_BUCKET, Key=filename.decode(
+            ) if isinstance(filename, bytes) else filename)
+            return response["Body"].read()
+        except ClientError as e:
+            print(f"Error downloading {filename} from S3: {e}")
+            return None
+    return await loop.run_in_executor(None, _download)
+
+
+async def main():
     while True:
-        # Block until a filename is available in the Redis queue
-        filename = redis_client.rpop(REDIS_QUEUE)
+        loop = asyncio.get_event_loop()
+        filename = await loop.run_in_executor(None, redis_client.rpop, REDIS_QUEUE)
         if filename:
-            file_path = os.path.join(UPLOAD_DIR, filename)
-            if os.path.isfile(file_path):
-                rca_result = process_log(file_path)
-                save_rca_to_db(rca_result)
-                # Notify ready-for-rca queue
+            file_content = await get_file_from_s3(filename)
+            if file_content is not None:
+                rca_result = await process_log(filename, file_content)
+                await save_rca_to_db(rca_result)
                 try:
-                    redis_client.lpush(READY_FOR_RCA_QUEUE, filename)
+                    await loop.run_in_executor(None, redis_client.lpush, READY_FOR_RCA_QUEUE, filename)
                     print(f"Notified {READY_FOR_RCA_QUEUE} for {filename}")
                 except Exception as redis_exc:
                     print(f"Failed to notify ready-for-rca queue: {redis_exc}")
             else:
-                print(f"File not found: {file_path}")
+                print(f"File {filename} could not be downloaded from S3.")
         else:
-            time.sleep(2)
+            await asyncio.sleep(2)
+
+
+# FastAPI endpoint to process a log file from S3
+@app.post("/process-log")
+async def process_log_endpoint(request: LogRequest):
+    file_content = await get_file_from_s3(request.filename)
+    if file_content is None:
+        raise HTTPException(status_code=404, detail=f"File {request.filename} not found in S3 bucket.")
+    result = await process_log(request.filename, file_content)
+    return result
 
 if __name__ == "__main__":
-    main()
+    import sys
+    if len(sys.argv) > 1 and sys.argv[1] == "serve":
+        uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=True)
+    else:
+        asyncio.run(main())

bert_pytorch/dataset/dataset.py

@@ -0,0 +1,131 @@
+from torch.utils.data import Dataset
+import tqdm
+import torch
+import random
+import numpy as np
+
+class BERTDataset(Dataset):
+    def __init__(self, corpus_path, vocab, seq_len, corpus_lines=None, encoding="utf-8", on_memory=True, predict_mode=False):
+        self.vocab = vocab
+        self.seq_len = seq_len
+
+        self.on_memory = on_memory
+        self.corpus_lines = corpus_lines
+        self.corpus_path = corpus_path
+        self.encoding = encoding
+
+        self.predict_mode = predict_mode
+        self.lines = corpus_path
+        self.corpus_lines = len(self.lines)
+
+        if not on_memory:
+            self.file = open(corpus_path, "r", encoding=encoding)
+            self.random_file = open(corpus_path, "r", encoding=encoding)
+
+            for _ in range(random.randint(self.corpus_lines if self.corpus_lines < 1000 else 1000)):
+                self.random_file.__next__()
+
+    def __len__(self):
+        return self.corpus_lines
+
+    def __getitem__(self, item):
+        t1, t2, is_next_label = self.random_sent(item)
+        t1_random, t1_label = self.random_word(t1)
+        t2_random, t2_label = self.random_word(t2)
+
+        # [CLS] tag = SOS tag, [SEP] tag = EOS tag
+        t1 = [self.vocab.sos_index] + t1_random + [self.vocab.eos_index]
+        t2 = t2_random + [self.vocab.eos_index]
+
+        t1_label = [self.vocab.pad_index] + t1_label + [self.vocab.pad_index]
+        t2_label = t2_label + [self.vocab.pad_index]
+
+        segment_label = ([1 for _ in range(len(t1))] + [2 for _ in range(len(t2))])[:self.seq_len]
+        bert_input = (t1 + t2)[:self.seq_len]
+        bert_label = (t1_label + t2_label)[:self.seq_len]
+
+        padding = [self.vocab.pad_index for _ in range(self.seq_len - len(bert_input))]
+        bert_input.extend(padding), bert_label.extend(padding), segment_label.extend(padding)
+
+        output = {"bert_input": bert_input,
+                  "bert_label": bert_label,
+                  "segment_label": segment_label,
+                  "is_next": is_next_label}
+
+        return {key: torch.tensor(value) for key, value in output.items()}
+
+    def random_word(self, sentence):
+        tokens = list(sentence)
+        output_label = []
+
+        for i, token in enumerate(tokens):
+            prob = random.random()
+            # replace 15% of tokens in a sequence to a masked token
+            if prob < 0.15:
+                if self.predict_mode:
+                    tokens[i] = self.vocab.mask_index
+                    output_label.append(self.vocab.stoi.get(token, self.vocab.unk_index))
+                    continue
+
+                prob /= 0.15
+
+                # 80% randomly change token to mask token
+                if prob < 0.8:
+                    tokens[i] = self.vocab.mask_index
+
+                # 10% randomly change token to random token
+                elif prob < 0.9:
+                    tokens[i] = random.randrange(len(self.vocab))
+
+                # 10% randomly change token to current token
+                else:
+                    tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index)
+
+                output_label.append(self.vocab.stoi.get(token, self.vocab.unk_index))
+
+            else:
+                tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index)
+                output_label.append(0)
+
+        return tokens, output_label
+
+    def random_sent(self, index):
+        t1, t2 = self.get_corpus_line(index)
+
+        if self.predict_mode:
+            return t1, t2, 1
+
+        # output_text, label(isNotNext:0, isNext:1)
+        if random.random() > 0.5:
+            return t1, t2, 1
+        else:
+            return t1, self.get_random_line(), 0
+
+    def get_corpus_line(self, item):
+        if self.on_memory:
+            return self.lines[item][0], self.lines[item][1]
+        else:
+            line = self.file.__next__()
+            if line is None:
+                self.file.close()
+                self.file = open(self.corpus_path, "r", encoding=self.encoding)
+                line = self.file.__next__()
+
+            t1, t2 = line[:-1].split("\t")
+            return t1, t2
+
+    def get_random_line(self):
+        if self.on_memory:
+            return self.lines[random.randrange(len(self.lines))][1]
+
+        line = self.file.__next__()
+        if line is None:
+            self.file.close()
+            self.file = open(self.corpus_path, "r", encoding=self.encoding)
+            for _ in range(random.randint(self.corpus_lines if self.corpus_lines < 1000 else 1000)):
+                self.random_file.__next__()
+            line = self.random_file.__next__()
+        return line[:-1].split("\t")[1]
+
+
+

bert_pytorch/dataset/log_dataset.py

@@ -0,0 +1,134 @@
+from torch.utils.data import Dataset
+import torch
+import random
+import numpy as np
+from collections import defaultdict
+
+class LogDataset(Dataset):
+    def __init__(self, log_corpus, time_corpus, vocab, seq_len, corpus_lines=None, encoding="utf-8", on_memory=True, predict_mode=False, mask_ratio=0.15):
+        """
+
+        :param corpus: log sessions/line
+        :param vocab: log events collection including pad, ukn ...
+        :param seq_len: max sequence length
+        :param corpus_lines: number of log sessions
+        :param encoding:
+        :param on_memory:
+        :param predict_mode: if predict
+        """
+        self.vocab = vocab
+        self.seq_len = seq_len
+
+        self.on_memory = on_memory
+        self.encoding = encoding
+
+        self.predict_mode = predict_mode
+        self.log_corpus = log_corpus
+        self.time_corpus = time_corpus
+        self.corpus_lines = len(log_corpus)
+
+        self.mask_ratio = mask_ratio
+
+    def __len__(self):
+        return self.corpus_lines
+
+    def __getitem__(self, idx):
+        k, t = self.log_corpus[idx], self.time_corpus[idx]
+
+        k_masked, k_label, t_masked, t_label = self.random_item(k, t)
+
+        # [CLS] tag = SOS tag, [SEP] tag = EOS tag
+        k = [self.vocab.sos_index] + k_masked
+        k_label = [self.vocab.pad_index] + k_label
+        # k_label = [self.vocab.sos_index] + k_label
+
+        t = [0] + t_masked
+        t_label = [self.vocab.pad_index] + t_label
+
+        return k, k_label, t, t_label
+
+    def random_item(self, k, t):
+        tokens = list(k)
+        output_label = []
+
+        time_intervals = list(t)
+        time_label = []
+
+        for i, token in enumerate(tokens):
+            time_int = time_intervals[i]
+            prob = random.random()
+            # replace 15% of tokens in a sequence to a masked token
+            if prob < self.mask_ratio:
+                # raise AttributeError("no mask in visualization")
+
+                if self.predict_mode:
+                    tokens[i] = self.vocab.mask_index
+                    output_label.append(self.vocab.stoi.get(token, self.vocab.unk_index))
+
+                    time_label.append(time_int)
+                    time_intervals[i] = 0
+                    continue
+
+                prob /= self.mask_ratio
+
+                # 80% randomly change token to mask token
+                if prob < 0.8:
+                    tokens[i] = self.vocab.mask_index
+
+                # 10% randomly change token to random token
+                elif prob < 0.9:
+                    tokens[i] = random.randrange(len(self.vocab))
+
+                # 10% randomly change token to current token
+                else:
+                    tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index)
+
+                output_label.append(self.vocab.stoi.get(token, self.vocab.unk_index))
+
+                time_intervals[i] = 0  # time mask value = 0
+                time_label.append(time_int)
+
+            else:
+                tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index)
+                output_label.append(0)
+                time_label.append(0)
+
+        return tokens, output_label, time_intervals, time_label
+
+    def collate_fn(self, batch, percentile=100, dynamical_pad=True):
+        lens = [len(seq[0]) for seq in batch]
+
+        # find the max len in each batch
+        if dynamical_pad:
+            # dynamical padding
+            seq_len = int(np.percentile(lens, percentile))
+            if self.seq_len is not None:
+                seq_len = min(seq_len, self.seq_len)
+        else:
+            # fixed length padding
+            seq_len = self.seq_len
+
+        output = defaultdict(list)
+        for seq in batch:
+            bert_input = seq[0][:seq_len]
+            bert_label = seq[1][:seq_len]
+            time_input = seq[2][:seq_len]
+            time_label = seq[3][:seq_len]
+
+            padding = [self.vocab.pad_index for _ in range(seq_len - len(bert_input))]
+            bert_input.extend(padding), bert_label.extend(padding), time_input.extend(padding), time_label.extend(
+                padding)
+
+            time_input = np.array(time_input)[:, np.newaxis]
+            output["bert_input"].append(bert_input)
+            output["bert_label"].append(bert_label)
+            output["time_input"].append(time_input)
+            output["time_label"].append(time_label)
+
+        output["bert_input"] = torch.tensor(output["bert_input"], dtype=torch.long)
+        output["bert_label"] = torch.tensor(output["bert_label"], dtype=torch.long)
+        output["time_input"] = torch.tensor(output["time_input"], dtype=torch.float)
+        output["time_label"] = torch.tensor(output["time_label"], dtype=torch.float)
+
+        return output
+

bert_pytorch/dataset/sample.py

@@ -0,0 +1,117 @@
+from tqdm import tqdm
+import numpy as np
+from sklearn.model_selection import train_test_split
+
+
+def generate_pairs(line, window_size):
+    line = np.array(line)
+    line = line[:, 0]
+
+    seqs = []
+    for i in range(0, len(line), window_size):
+        seq = line[i:i + window_size]
+        seqs.append(seq)
+    seqs += []
+    seq_pairs = []
+    for i in range(1, len(seqs)):
+        seq_pairs.append([seqs[i - 1], seqs[i]])
+    return seqs
+
+
+def fixed_window(line, window_size, adaptive_window, seq_len=None, min_len=0):
+    line = [ln.split(",") for ln in line.split()]
+
+    # filter the line/session shorter than 10
+    if len(line) < min_len:
+        return [], []
+
+    # max seq len
+    if seq_len is not None:
+        line = line[:seq_len]
+
+    if adaptive_window:
+        window_size = len(line)
+
+    line = np.array(line)
+
+    # if time duration exists in data
+    if line.shape[1] == 2:
+        tim = line[:,1].astype(float)
+        line = line[:, 0]
+
+        # the first time duration of a session should be 0, so max is window_size(mins) * 60
+        tim[0] = 0
+    else:
+        line = line.squeeze()
+        # if time duration doesn't exist, then create a zero array for time
+        tim = np.zeros(line.shape)
+
+    logkey_seqs = []
+    time_seq = []
+    for i in range(0, len(line), window_size):
+        logkey_seqs.append(line[i:i + window_size])
+        time_seq.append(tim[i:i + window_size])
+
+    return logkey_seqs, time_seq
+
+
+def generate_train_valid(data_path, window_size=20, adaptive_window=True,
+                         sample_ratio=1, valid_size=0.1, output_path=None,
+                         scale=None, scale_path=None, seq_len=None, min_len=0):
+    with open(data_path, 'r') as f:
+        data_iter = f.readlines()
+
+    num_session = int(len(data_iter) * sample_ratio)
+    # only even number of samples, or drop_last=True in DataLoader API
+    # coz in parallel computing in CUDA, odd number of samples reports issue when merging the result
+    # num_session += num_session % 2
+
+    test_size = int(min(num_session, len(data_iter)) * valid_size)
+    # only even number of samples
+    # test_size += test_size % 2
+
+    print("before filtering short session")
+    print("train size ", int(num_session - test_size))
+    print("valid size ", int(test_size))
+    print("="*40)
+
+    logkey_seq_pairs = []
+    time_seq_pairs = []
+    session = 0
+    for line in tqdm(data_iter):
+        if session >= num_session:
+            break
+        session += 1
+
+        logkeys, times = fixed_window(line, window_size, adaptive_window, seq_len, min_len)
+        logkey_seq_pairs += logkeys
+        time_seq_pairs += times
+
+    logkey_seq_pairs = np.array(logkey_seq_pairs, dtype=object)
+    time_seq_pairs = np.array(time_seq_pairs, dtype=object)
+
+    logkey_trainset, logkey_validset, time_trainset, time_validset = train_test_split(logkey_seq_pairs,
+                                                                                      time_seq_pairs,
+                                                                                      test_size=test_size,
+                                                                                      random_state=1234)
+
+    # sort seq_pairs by seq len
+    train_len = list(map(len, logkey_trainset))
+    valid_len = list(map(len, logkey_validset))
+
+    train_sort_index = np.argsort(-1 * np.array(train_len))
+    valid_sort_index = np.argsort(-1 * np.array(valid_len))
+
+    logkey_trainset = logkey_trainset[train_sort_index]
+    logkey_validset = logkey_validset[valid_sort_index]
+
+    time_trainset = time_trainset[train_sort_index]
+    time_validset = time_validset[valid_sort_index]
+
+    print("="*40)
+    print("Num of train seqs", len(logkey_trainset))
+    print("Num of valid seqs", len(logkey_validset))
+    print("="*40)
+
+    return logkey_trainset, logkey_validset, time_trainset, time_validset
+

bert_pytorch/dataset/utils.py

@@ -0,0 +1,19 @@
+import random
+import os
+import numpy as np
+import torch
+
+
+def save_parameters(options, filename):
+    with open(filename, "w+") as f:
+        for key in options.keys():
+            f.write("{}: {}\n".format(key, options[key]))
+
+# https://gist.github.com/KirillVladimirov/005ec7f762293d2321385580d3dbe335
+def seed_everything(seed=1234):
+    random.seed(seed)
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    # torch.cuda.manual_seed(seed)
+    # torch.backends.cudnn.deterministic = True

bert_pytorch/dataset/vocab.py

@@ -0,0 +1,169 @@
+import pickle
+import tqdm
+from collections import Counter
+import sys
+sys.path.append("../")
+
+class TorchVocab(object):
+    """Defines a vocabulary object that will be used to numericalize a field.
+    Attributes:
+        freqs: A collections.Counter object holding the frequencies of tokens
+            in the data used to build the Vocab.
+        stoi: A collections.defaultdict instance mapping token strings to
+            numerical identifiers.
+        itos: A list of token strings indexed by their numerical identifiers.
+    """
+
+    def __init__(self, counter, max_size=None, min_freq=1, specials=['<pad>', '<oov>'],
+                 vectors=None, unk_init=None, vectors_cache=None):
+        """Create a Vocab object from a collections.Counter.
+        Arguments:
+            counter: collections.Counter object holding the frequencies of
+                each value found in the data.
+            max_size: The maximum size of the vocabulary, or None for no
+                maximum. Default: None.
+            min_freq: The minimum frequency needed to include a token in the
+                vocabulary. Values less than 1 will be set to 1. Default: 1.
+            specials: The list of special tokens (e.g., padding or eos) that
+                will be prepended to the vocabulary in addition to an <unk>
+                token. Default: ['<pad>']
+            vectors: One of either the available pretrained vectors
+                or custom pretrained vectors (see Vocab.load_vectors);
+                or a list of aforementioned vectors
+            unk_init (callback): by default, initialize out-of-vocabulary word vectors
+                to zero vectors; can be any function that takes in a Tensor and
+                returns a Tensor of the same size. Default: torch.Tensor.zero_
+            vectors_cache: directory for cached vectors. Default: '.vector_cache'
+        """
+        self.freqs = counter
+        counter = counter.copy()
+        min_freq = max(min_freq, 1)
+
+        self.itos = list(specials)
+        # frequencies of special tokens are not counted when building vocabulary
+        # in frequency order
+        for tok in specials:
+            del counter[tok]
+
+        max_size = None if max_size is None else max_size + len(self.itos)
+
+        # sort by frequency, then alphabetically
+        words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0])
+        words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True)
+
+        for word, freq in words_and_frequencies:
+            if freq < min_freq or len(self.itos) == max_size:
+                break
+            self.itos.append(word)
+
+        # stoi is simply a reverse dict for itos
+        self.stoi = {tok: i for i, tok in enumerate(self.itos)}
+
+        self.vectors = None
+        if vectors is not None:
+            self.load_vectors(vectors, unk_init=unk_init, cache=vectors_cache)
+        else:
+            assert unk_init is None and vectors_cache is None
+
+    def __eq__(self, other):
+        if self.freqs != other.freqs:
+            return False
+        if self.stoi != other.stoi:
+            return False
+        if self.itos != other.itos:
+            return False
+        if self.vectors != other.vectors:
+            return False
+        return True
+
+    def __len__(self):
+        return len(self.itos)
+
+    def vocab_rerank(self):
+        self.stoi = {word: i for i, word in enumerate(self.itos)}
+
+    def extend(self, v, sort=False):
+        words = sorted(v.itos) if sort else v.itos
+        for w in words:
+            if w not in self.stoi:
+                self.itos.append(w)
+                self.stoi[w] = len(self.itos) - 1
+
+
+class Vocab(TorchVocab):
+    def __init__(self, counter, max_size=None, min_freq=1):
+        self.pad_index = 0
+        self.unk_index = 1
+        self.eos_index = 2
+        self.sos_index = 3
+        self.mask_index = 4
+        super().__init__(counter, specials=["<pad>", "<unk>", "<eos>", "<sos>", "<mask>"],
+                         max_size=max_size, min_freq=min_freq)
+
+    def to_seq(self, sentece, seq_len, with_eos=False, with_sos=False) -> list:
+        pass
+
+    def from_seq(self, seq, join=False, with_pad=False):
+        pass
+
+    @staticmethod
+    def load_vocab(vocab_path: str) -> 'Vocab':
+        with open(vocab_path, "rb") as f:
+            return pickle.load(f)
+
+    def save_vocab(self, vocab_path):
+        with open(vocab_path, "wb") as f:
+            pickle.dump(self, f)
+
+
+# Building Vocab with text files
+class WordVocab(Vocab):
+    def __init__(self, texts, max_size=None, min_freq=1):
+        print("Building Vocab")
+        counter = Counter()
+        for line in tqdm.tqdm(texts):
+            if isinstance(line, list):
+                words = line
+            else:
+                words = line.replace("\n", "").replace("\t", "").split()
+
+            for word in words:
+                counter[word] += 1
+        super().__init__(counter, max_size=max_size, min_freq=min_freq)
+
+    def to_seq(self, sentence, seq_len=None, with_eos=False, with_sos=False, with_len=False):
+        if isinstance(sentence, str):
+            sentence = sentence.split()
+
+        seq = [self.stoi.get(word, self.unk_index) for word in sentence]
+
+        if with_eos:
+            seq += [self.eos_index]  # this would be index 1
+        if with_sos:
+            seq = [self.sos_index] + seq
+
+        origin_seq_len = len(seq)
+
+        if seq_len is None:
+            pass
+        elif len(seq) <= seq_len:
+            seq += [self.pad_index for _ in range(seq_len - len(seq))]
+        else:
+            seq = seq[:seq_len]
+
+        return (seq, origin_seq_len) if with_len else seq
+
+    def from_seq(self, seq, join=False, with_pad=False):
+        words = [self.itos[idx]
+                 if idx < len(self.itos)
+                 else "<%d>" % idx
+                 for idx in seq
+                 if not with_pad or idx != self.pad_index]
+
+        return " ".join(words) if join else words
+
+    @staticmethod
+    def load_vocab(vocab_path: str) -> 'WordVocab':
+        with open(vocab_path, "rb") as f:
+            return pickle.load(f)
+

bert_pytorch/model/bert.py

@@ -0,0 +1,49 @@
+import torch.nn as nn
+import torch
+
+from .transformer import TransformerBlock
+from .embedding import BERTEmbedding
+
+class BERT(nn.Module):
+    """
+    BERT model : Bidirectional Encoder Representations from Transformers.
+    """
+
+    def __init__(self, vocab_size, max_len=512, hidden=768, n_layers=12, attn_heads=12, dropout=0.1, is_logkey=True, is_time=False):
+        """
+        :param vocab_size: vocab_size of total words
+        :param hidden: BERT model hidden size
+        :param n_layers: numbers of Transformer blocks(layers)
+        :param attn_heads: number of attention heads
+        :param dropout: dropout rate
+        """
+
+        super().__init__()
+        self.hidden = hidden
+        self.n_layers = n_layers
+        self.attn_heads = attn_heads
+
+        # paper noted they used 4*hidden_size for ff_network_hidden_size
+        self.feed_forward_hidden = hidden * 2
+
+        # embedding for BERT, sum of positional, segment, token embeddings
+        self.embedding = BERTEmbedding(vocab_size=vocab_size, embed_size=hidden, max_len=max_len, is_logkey=is_logkey, is_time=is_time)
+
+        # multi-layers transformer blocks, deep network
+        self.transformer_blocks = nn.ModuleList(
+            [TransformerBlock(hidden, attn_heads, hidden * 2, dropout) for _ in range(n_layers)])
+
+
+    def forward(self, x, segment_info=None, time_info=None):
+        # attention masking for padded token
+        # torch.ByteTensor([batch_size, 1, seq_len, seq_len)
+        mask = (x > 0).unsqueeze(1).repeat(1, x.size(1), 1).unsqueeze(1)
+
+        # embedding the indexed sequence to sequence of vectors
+        x = self.embedding(x, segment_info, time_info)
+
+        # running over multiple transformer blocks
+        for transformer in self.transformer_blocks:
+            x = transformer.forward(x, mask)
+
+        return x

bert_pytorch/model/embedding/bert.py

@@ -0,0 +1,42 @@
+import torch.nn as nn
+import torch
+from .token import TokenEmbedding
+from .position import PositionalEmbedding
+from .segment import SegmentEmbedding
+from .time_embed import TimeEmbedding
+
+class BERTEmbedding(nn.Module):
+    """
+    BERT Embedding which is consisted with under features
+        1. TokenEmbedding : normal embedding matrix
+        2. PositionalEmbedding : adding positional information using sin, cos
+        2. SegmentEmbedding : adding sentence segment info, (sent_A:1, sent_B:2)
+
+        sum of all these features are output of BERTEmbedding
+    """
+
+    def __init__(self, vocab_size, embed_size, max_len, dropout=0.1, is_logkey=True, is_time=False):
+        """
+        :param vocab_size: total vocab size
+        :param embed_size: embedding size of token embedding
+        :param dropout: dropout rate
+        """
+        super().__init__()
+        self.token = TokenEmbedding(vocab_size=vocab_size, embed_size=embed_size)
+        self.position = PositionalEmbedding(d_model=self.token.embedding_dim, max_len=max_len)
+        self.segment = SegmentEmbedding(embed_size=self.token.embedding_dim)
+        self.time_embed = TimeEmbedding(embed_size=self.token.embedding_dim)
+        self.dropout = nn.Dropout(p=dropout)
+        self.embed_size = embed_size
+        self.is_logkey = is_logkey
+        self.is_time = is_time
+
+    def forward(self, sequence, segment_label=None, time_info=None):
+        x = self.position(sequence)
+        # if self.is_logkey:
+        x = x + self.token(sequence)
+        if segment_label is not None:
+            x = x + self.segment(segment_label)
+        if self.is_time:
+            x = x + self.time_embed(time_info)
+        return self.dropout(x)

bert_pytorch/model/embedding/position.py

@@ -0,0 +1,25 @@
+import torch.nn as nn
+import torch
+import math
+
+
+class PositionalEmbedding(nn.Module):
+
+    def __init__(self, d_model, max_len=512):
+        super().__init__()
+
+        # Compute the positional encodings once in log space.
+        pe = torch.zeros(max_len, d_model).float()
+        pe.require_grad = False
+
+        position = torch.arange(0, max_len).float().unsqueeze(1)
+        div_term = (torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)).exp()
+
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+
+        pe = pe.unsqueeze(0)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x):
+        return self.pe[:, :x.size(1)]

bert_pytorch/model/embedding/segment.py

@@ -0,0 +1,6 @@
+import torch.nn as nn
+
+
+class SegmentEmbedding(nn.Embedding):
+    def __init__(self, embed_size=512):
+        super().__init__(3, embed_size, padding_idx=0)

bert_pytorch/model/embedding/time_embed.py

@@ -0,0 +1,10 @@
+import torch.nn as nn
+
+
+class TimeEmbedding(nn.Module):
+    def __init__(self, embed_size=512):
+        super().__init__()
+        self.time_embed = nn.Linear(1, embed_size)
+
+    def forward(self, time_interval):
+        return self.time_embed(time_interval)

bert_pytorch/model/embedding/token.py

@@ -0,0 +1,6 @@
+import torch.nn as nn
+
+
+class TokenEmbedding(nn.Embedding):
+    def __init__(self, vocab_size, embed_size=512):
+        super().__init__(vocab_size, embed_size, padding_idx=0)

bert_pytorch/model/language_model.py

@@ -0,0 +1,61 @@
+import torch.nn as nn
+
+from .bert import BERT
+
+
+class BERTLM(nn.Module):
+    """
+    BERT Language Model
+    Next Sentence Prediction Model + Masked Language Model
+    """
+
+    def __init__(self, bert: BERT, vocab_size):
+        """
+        :param bert: BERT model which should be trained
+        :param vocab_size: total vocab size for masked_lm
+        """
+
+        super().__init__()
+        self.bert = bert
+        self.next_sentence = NextSentencePrediction(self.bert.hidden)
+        self.mask_lm = MaskedLanguageModel(self.bert.hidden, vocab_size)
+
+    def forward(self, x, segment_label):
+        x = self.bert(x, segment_label)
+        return self.next_sentence(x), self.mask_lm(x)
+
+
+class NextSentencePrediction(nn.Module):
+    """
+    2-class classification model : is_next, is_not_next
+    """
+
+    def __init__(self, hidden):
+        """
+        :param hidden: BERT model output size
+        """
+        super().__init__()
+        self.linear = nn.Linear(hidden, 2)
+        self.softmax = nn.LogSoftmax(dim=-1)
+
+    def forward(self, x):
+        return self.softmax(self.linear(x[:, 0]))
+
+
+class MaskedLanguageModel(nn.Module):
+    """
+    predicting origin token from masked input sequence
+    n-class classification problem, n-class = vocab_size
+    """
+
+    def __init__(self, hidden, vocab_size):
+        """
+        :param hidden: output size of BERT model
+        :param vocab_size: total vocab size
+        """
+        super().__init__()
+        self.linear = nn.Linear(hidden, vocab_size)
+        self.softmax = nn.LogSoftmax(dim=-1)
+
+    def forward(self, x):
+        return self.softmax(self.linear(x))

bert_pytorch/model/log_model.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+from .bert import BERT
+
+class BERTLog(nn.Module):
+    """
+    BERT Log Model
+    """
+
+    def __init__(self, bert: BERT, vocab_size):
+        """
+        :param bert: BERT model which should be trained
+        :param vocab_size: total vocab size for masked_lm
+        """
+        super().__init__()
+        self.bert = bert
+        self.mask_lm = MaskedLogModel(self.bert.hidden, vocab_size)
+        self.time_lm = TimeLogModel(self.bert.hidden)
+        # self.fnn_cls = LinearCLS(self.bert.hidden)
+        # self.cls_lm = LogClassifier(self.bert.hidden)
+
+    def forward(self, x, time_info):
+        x = self.bert(x, time_info=time_info)  # [batch, seq_len, hidden]
+
+        cls_output = x[:, 0]  # [CLS] token vector from BERT
+
+        return {
+            "logkey_output": self.mask_lm(x),    # [batch, seq_len, vocab_size]
+            "time_output": self.time_lm(x),      # optional
+            "cls_output": cls_output,            # [batch, hidden]
+            "cls_fnn_output": None,              # unused for now
+            "token_embeddings": x[0]             # [seq_len, hidden] for first batch element
+        }
+
+
+class MaskedLogModel(nn.Module):
+    """
+    Predicting original token from masked input sequence
+    """
+
+    def __init__(self, hidden, vocab_size):
+        super().__init__()
+        self.linear = nn.Linear(hidden, vocab_size)
+        self.softmax = nn.LogSoftmax(dim=-1)
+
+    def forward(self, x):
+        return self.softmax(self.linear(x))
+
+
+class TimeLogModel(nn.Module):
+    def __init__(self, hidden, time_size=1):
+        super().__init__()
+        self.linear = nn.Linear(hidden, time_size)
+
+    def forward(self, x):
+        return self.linear(x)
+
+
+class LogClassifier(nn.Module):
+    def __init__(self, hidden):
+        super().__init__()
+        self.linear = nn.Linear(hidden, hidden)
+
+    def forward(self, cls):
+        return self.linear(cls)
+
+
+class LinearCLS(nn.Module):
+    def __init__(self, hidden):
+        super().__init__()
+        self.linear = nn.Linear(hidden, hidden)
+
+    def forward(self, x):
+        return self.linear(x)

bert_pytorch/model/transformer.py

@@ -0,0 +1,31 @@
+import torch.nn as nn
+
+from .attention import MultiHeadedAttention
+from .utils import SublayerConnection, PositionwiseFeedForward
+
+
+class TransformerBlock(nn.Module):
+    """
+    Bidirectional Encoder = Transformer (self-attention)
+    Transformer = MultiHead_Attention + Feed_Forward with sublayer connection
+    """
+
+    def __init__(self, hidden, attn_heads, feed_forward_hidden, dropout):
+        """
+        :param hidden: hidden size of transformer
+        :param attn_heads: head sizes of multi-head attention
+        :param feed_forward_hidden: feed_forward_hidden, usually 4*hidden_size
+        :param dropout: dropout rate
+        """
+
+        super().__init__()
+        self.attention = MultiHeadedAttention(h=attn_heads, d_model=hidden)
+        self.feed_forward = PositionwiseFeedForward(d_model=hidden, d_ff=feed_forward_hidden, dropout=dropout)
+        self.input_sublayer = SublayerConnection(size=hidden, dropout=dropout)
+        self.output_sublayer = SublayerConnection(size=hidden, dropout=dropout)
+        self.dropout = nn.Dropout(p=dropout)
+
+    def forward(self, x, mask):
+        x = self.input_sublayer(x, lambda _x: self.attention.forward(_x, _x, _x, mask=mask))
+        x = self.output_sublayer(x, self.feed_forward)
+        return self.dropout(x)

bert_pytorch/predict_log.py

@@ -0,0 +1,290 @@
+import numpy as np
+import scipy.stats as stats
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pickle
+import time
+import torch
+from tqdm import tqdm
+from torch.utils.data import DataLoader
+
+from bert_pytorch.dataset import WordVocab
+from bert_pytorch.dataset import LogDataset
+from bert_pytorch.dataset.sample import fixed_window
+
+
+def compute_anomaly(results, params, seq_threshold=0.5):
+    is_logkey = params["is_logkey"]
+    is_time = params["is_time"]
+    total_errors = 0
+    for seq_res in results:
+        # label pairs as anomaly when over half of masked tokens are undetected
+        if (is_logkey and seq_res["undetected_tokens"] > seq_res["masked_tokens"] * seq_threshold) or \
+                (is_time and seq_res["num_error"]> seq_res["masked_tokens"] * seq_threshold) or \
+                (params["hypersphere_loss_test"] and seq_res["deepSVDD_label"]):
+            total_errors += 1
+    return total_errors
+
+
+def find_best_threshold(test_normal_results, test_abnormal_results, params, th_range, seq_range):
+    best_result = [0] * 9
+    for seq_th in seq_range:
+        FP = compute_anomaly(test_normal_results, params, seq_th)
+        TP = compute_anomaly(test_abnormal_results, params, seq_th)
+
+        if TP == 0:
+            continue
+
+        TN = len(test_normal_results) - FP
+        FN = len(test_abnormal_results) - TP
+        P = 100 * TP / (TP + FP)
+        R = 100 * TP / (TP + FN)
+        F1 = 2 * P * R / (P + R)
+
+        if F1 > best_result[-1]:
+            best_result = [0, seq_th, FP, TP, TN, FN, P, R, F1]
+    return best_result
+
+
+class Predictor():
+    def __init__(self, options):
+        self.model_path = options["model_path"]
+        self.vocab_path = options["vocab_path"]
+        self.device = options["device"]
+        self.window_size = options["window_size"]
+        self.adaptive_window = options["adaptive_window"]
+        self.seq_len = options["seq_len"]
+        self.corpus_lines = options["corpus_lines"]
+        self.on_memory = options["on_memory"]
+        self.batch_size = options["batch_size"]
+        self.num_workers = options["num_workers"]
+        self.num_candidates = options["num_candidates"]
+        self.output_dir = options["output_dir"]
+        self.model_dir = options["model_dir"]
+        self.gaussian_mean = options["gaussian_mean"]
+        self.gaussian_std = options["gaussian_std"]
+
+        self.is_logkey = options["is_logkey"]
+        self.is_time = options["is_time"]
+        self.scale_path = options["scale_path"]
+
+        self.hypersphere_loss = options["hypersphere_loss"]
+        self.hypersphere_loss_test = options["hypersphere_loss_test"]
+
+        self.lower_bound = self.gaussian_mean - 3 * self.gaussian_std
+        self.upper_bound = self.gaussian_mean + 3 * self.gaussian_std
+
+        self.center = None
+        self.radius = None
+        self.test_ratio = options["test_ratio"]
+        self.mask_ratio = options["mask_ratio"]
+        self.min_len=options["min_len"]
+
+    def detect_logkey_anomaly(self, masked_output, masked_label):
+        num_undetected_tokens = 0
+        output_maskes = []
+        for i, token in enumerate(masked_label):
+            # output_maskes.append(torch.argsort(-masked_output[i])[:30].cpu().numpy()) # extract top 30 candidates for mask labels
+
+            if token not in torch.argsort(-masked_output[i])[:self.num_candidates]:
+                num_undetected_tokens += 1
+
+        return num_undetected_tokens, [output_maskes, masked_label.cpu().numpy()]
+
+    @staticmethod
+    def generate_test(output_dir, file_name, window_size, adaptive_window, seq_len, scale, min_len):
+        """
+        :return: log_seqs: num_samples x session(seq)_length, tim_seqs: num_samples x session_length
+        """
+        log_seqs = []
+        tim_seqs = []
+        with open(output_dir + file_name, "r") as f:
+            for idx, line in tqdm(enumerate(f.readlines())):
+                #if idx > 40: break
+                log_seq, tim_seq = fixed_window(line, window_size,
+                                                adaptive_window=adaptive_window,
+                                                seq_len=seq_len, min_len=min_len)
+                if len(log_seq) == 0:
+                    continue
+
+                # if scale is not None:
+                #     times = tim_seq
+                #     for i, tn in enumerate(times):
+                #         tn = np.array(tn).reshape(-1, 1)
+                #         times[i] = scale.transform(tn).reshape(-1).tolist()
+                #     tim_seq = times
+
+                log_seqs += log_seq
+                tim_seqs += tim_seq
+
+        # sort seq_pairs by seq len
+        log_seqs = np.array(log_seqs, dtype=object)
+        tim_seqs = np.array(tim_seqs, dtype=object)
+
+        test_len = list(map(len, log_seqs))
+        test_sort_index = np.argsort(-1 * np.array(test_len))
+
+        log_seqs = log_seqs[test_sort_index]
+        tim_seqs = tim_seqs[test_sort_index]
+
+        print(f"{file_name} size: {len(log_seqs)}")
+        return log_seqs, tim_seqs
+
+    def helper(self, model, output_dir, file_name, vocab, scale=None, error_dict=None):
+        total_results = []
+        total_errors = []
+        output_results = []
+        total_dist = []
+        output_cls = []
+        logkey_test, time_test = self.generate_test(output_dir, file_name, self.window_size, self.adaptive_window, self.seq_len, scale, self.min_len)
+
+        # use 1/10 test data
+        if self.test_ratio != 1:
+            num_test = len(logkey_test)
+            rand_index = torch.randperm(num_test)
+            rand_index = rand_index[:int(num_test * self.test_ratio)] if isinstance(self.test_ratio, float) else rand_index[:self.test_ratio]
+            logkey_test, time_test = logkey_test[rand_index], time_test[rand_index]
+
+
+        seq_dataset = LogDataset(logkey_test, time_test, vocab, seq_len=self.seq_len,
+                                 corpus_lines=self.corpus_lines, on_memory=self.on_memory, predict_mode=True, mask_ratio=self.mask_ratio)
+
+        # use large batch size in test data
+        data_loader = DataLoader(seq_dataset, batch_size=self.batch_size, num_workers=self.num_workers,
+                                 collate_fn=seq_dataset.collate_fn)
+
+        for idx, data in enumerate(data_loader):
+            data = {key: value.to(self.device) for key, value in data.items()}
+
+            result = model(data["bert_input"], data["time_input"])
+
+            # mask_lm_output, mask_tm_output: batch_size x session_size x vocab_size
+            # cls_output: batch_size x hidden_size
+            # bert_label, time_label: batch_size x session_size
+            # in session, some logkeys are masked
+
+            mask_lm_output, mask_tm_output = result["logkey_output"], result["time_output"]
+            output_cls += result["cls_output"].tolist()
+
+            # dist = torch.sum((result["cls_output"] - self.hyper_center) ** 2, dim=1)
+            # when visualization no mask
+            # continue
+
+            # loop though each session in batch
+            for i in range(len(data["bert_label"])):
+                seq_results = {"num_error": 0,
+                               "undetected_tokens": 0,
+                               "masked_tokens": 0,
+                               "total_logkey": torch.sum(data["bert_input"][i] > 0).item(),
+                               "deepSVDD_label": 0
+                               }
+
+                mask_index = data["bert_label"][i] > 0
+                num_masked = torch.sum(mask_index).tolist()
+                seq_results["masked_tokens"] = num_masked
+
+                if self.is_logkey:
+                    num_undetected, output_seq = self.detect_logkey_anomaly(
+                        mask_lm_output[i][mask_index], data["bert_label"][i][mask_index])
+                    seq_results["undetected_tokens"] = num_undetected
+
+                    output_results.append(output_seq)
+
+                if self.hypersphere_loss_test:
+                    # detect by deepSVDD distance
+                    assert result["cls_output"][i].size() == self.center.size()
+                    # dist = torch.sum((result["cls_fnn_output"][i] - self.center) ** 2)
+                    dist = torch.sqrt(torch.sum((result["cls_output"][i] - self.center) ** 2))
+                    total_dist.append(dist.item())
+
+                    # user defined threshold for deepSVDD_label
+                    seq_results["deepSVDD_label"] = int(dist.item() > self.radius)
+                    #
+                    # if dist > 0.25:
+                    #     pass
+
+                if idx < 10 or idx % 1000 == 0:
+                    print(
+                        "{}, #time anomaly: {} # of undetected_tokens: {}, # of masked_tokens: {} , "
+                        "# of total logkey {}, deepSVDD_label: {} \n".format(
+                            file_name,
+                            seq_results["num_error"],
+                            seq_results["undetected_tokens"],
+                            seq_results["masked_tokens"],
+                            seq_results["total_logkey"],
+                            seq_results['deepSVDD_label']
+                        )
+                    )
+                total_results.append(seq_results)
+
+        # for time
+        # return total_results, total_errors
+
+        #for logkey
+        # return total_results, output_results
+
+        # for hypersphere distance
+        return total_results, output_cls
+
+    def predict(self):
+        model = torch.load(self.model_path, weights_only=False)
+        model.to(self.device)
+        model.eval()
+        print('model_path: {}'.format(self.model_path))
+
+        start_time = time.time()
+        vocab = WordVocab.load_vocab(self.vocab_path)
+
+        scale = None
+        error_dict = None
+        if self.is_time:
+            with open(self.scale_path, "rb") as f:
+                scale = pickle.load(f)
+
+            with open(self.model_dir + "error_dict.pkl", 'rb') as f:
+                error_dict = pickle.load(f)
+
+        if self.hypersphere_loss:
+            center_dict = torch.load(self.model_dir + "best_center.pt", weights_only=False)
+            self.center = center_dict["center"]
+            self.radius = center_dict["radius"]
+            # self.center = self.center.view(1,-1)
+
+
+        print("test normal predicting")
+        test_normal_results, test_normal_errors = self.helper(model, self.output_dir, "test_normal", vocab, scale, error_dict)
+
+        print("test abnormal predicting")
+        test_abnormal_results, test_abnormal_errors = self.helper(model, self.output_dir, "test_abnormal", vocab, scale, error_dict)
+
+        print("Saving test normal results")
+        with open(self.model_dir + "test_normal_results", "wb") as f:
+            pickle.dump(test_normal_results, f)
+
+        print("Saving test abnormal results")
+        with open(self.model_dir + "test_abnormal_results", "wb") as f:
+            pickle.dump(test_abnormal_results, f)
+
+        print("Saving test normal errors")
+        with open(self.model_dir + "test_normal_errors.pkl", "wb") as f:
+            pickle.dump(test_normal_errors, f)
+
+        print("Saving test abnormal results")
+        with open(self.model_dir + "test_abnormal_errors.pkl", "wb") as f:
+            pickle.dump(test_abnormal_errors, f)
+
+        params = {"is_logkey": self.is_logkey, "is_time": self.is_time, "hypersphere_loss": self.hypersphere_loss,
+                  "hypersphere_loss_test": self.hypersphere_loss_test}
+        best_th, best_seq_th, FP, TP, TN, FN, P, R, F1 = find_best_threshold(test_normal_results,
+                                                                            test_abnormal_results,
+                                                                            params=params,
+                                                                            th_range=np.arange(10),
+                                                                            seq_range=np.arange(0,1,0.1))
+
+        print("best threshold: {}, best threshold ratio: {}".format(best_th, best_seq_th))
+        print("TP: {}, TN: {}, FP: {}, FN: {}".format(TP, TN, FP, FN))
+        print('Precision: {:.2f}%, Recall: {:.2f}%, F1-measure: {:.2f}%'.format(P, R, F1))
+        elapsed_time = time.time() - start_time
+        print('elapsed_time: {}'.format(elapsed_time))
+
+

bert_pytorch/train_log.py

@@ -0,0 +1,222 @@
+import os
+import gc
+import torch
+import tqdm
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from torch.utils.data import DataLoader
+from bert_pytorch.model import BERT
+from bert_pytorch.trainer import BERTTrainer
+from bert_pytorch.dataset import LogDataset, WordVocab
+from bert_pytorch.dataset.sample import generate_train_valid
+from bert_pytorch.dataset.utils import save_parameters
+
+
+class Trainer():
+    def __init__(self, options):
+        self.device = options["device"]
+        self.model_dir = options["model_dir"]
+        self.model_path = options["model_path"]
+        self.vocab_path = options["vocab_path"]
+        self.output_path = options["output_dir"]
+        self.window_size = options["window_size"]
+        self.adaptive_window = options["adaptive_window"]
+        self.sample_ratio = options["train_ratio"]
+        self.valid_ratio = options["valid_ratio"]
+        self.seq_len = options["seq_len"]
+        self.max_len = options["max_len"]
+        self.corpus_lines = options["corpus_lines"]
+        self.on_memory = options["on_memory"]
+        self.batch_size = options["batch_size"]
+        self.num_workers = options["num_workers"]
+        self.lr = options["lr"]
+        self.adam_beta1 = options["adam_beta1"]
+        self.adam_beta2 = options["adam_beta2"]
+        self.adam_weight_decay = options["adam_weight_decay"]
+        self.with_cuda = options["with_cuda"]
+        self.cuda_devices = options["cuda_devices"]
+        self.log_freq = options["log_freq"]
+        self.epochs = options["epochs"]
+        self.hidden = options["hidden"]
+        self.layers = options["layers"]
+        self.attn_heads = options["attn_heads"]
+        self.is_logkey = options["is_logkey"]
+        self.is_time = options["is_time"]
+        self.scale = options["scale"]
+        self.scale_path = options["scale_path"]
+        self.n_epochs_stop = options["n_epochs_stop"]
+        self.hypersphere_loss = options["hypersphere_loss"]
+        self.mask_ratio = options["mask_ratio"]
+        self.min_len = options["min_len"]
+
+        print("Save options parameters")
+        save_parameters(options, self.model_dir + "parameters.txt")
+
+    def train(self):
+        print("Loading vocab", self.vocab_path)
+        vocab = WordVocab.load_vocab(self.vocab_path)
+        print("vocab Size: ", len(vocab))
+
+        print("\nLoading Train Dataset")
+        train_file_path = os.path.join(self.output_path, "train")
+        logkey_train, logkey_valid, time_train, time_valid = generate_train_valid(
+            train_file_path,
+            window_size=self.window_size,
+            adaptive_window=self.adaptive_window,
+            valid_size=self.valid_ratio,
+            sample_ratio=self.sample_ratio,
+            scale=self.scale,
+            scale_path=self.scale_path,
+            seq_len=self.seq_len,
+            min_len=self.min_len
+        )
+
+        train_dataset = LogDataset(
+            logkey_train, time_train, vocab,
+            seq_len=self.seq_len,
+            corpus_lines=self.corpus_lines,
+            on_memory=self.on_memory,
+            mask_ratio=self.mask_ratio
+        )
+
+        print("\nLoading valid Dataset")
+        valid_dataset = LogDataset(
+            logkey_valid, time_valid, vocab,
+            seq_len=self.seq_len,
+            on_memory=self.on_memory,
+            mask_ratio=self.mask_ratio
+        )
+
+        print("Creating Dataloader")
+        self.train_data_loader = DataLoader(
+            train_dataset,
+            batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            collate_fn=train_dataset.collate_fn,
+            drop_last=False
+        )
+
+        self.valid_data_loader = DataLoader(
+            valid_dataset,
+            batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            collate_fn=train_dataset.collate_fn,
+            drop_last=False
+        )
+
+        del train_dataset
+        del valid_dataset
+        del logkey_train
+        del logkey_valid
+        del time_train
+        del time_valid
+        gc.collect()
+
+        print("Building BERT model")
+        bert = BERT(
+            len(vocab),
+            max_len=self.max_len,
+            hidden=self.hidden,
+            n_layers=self.layers,
+            attn_heads=self.attn_heads,
+            is_logkey=self.is_logkey,
+            is_time=self.is_time
+        )
+
+        print("Creating BERT Trainer")
+        self.trainer = BERTTrainer(
+            bert, len(vocab),
+            train_dataloader=self.train_data_loader,
+            valid_dataloader=self.valid_data_loader,
+            lr=self.lr,
+            betas=(self.adam_beta1, self.adam_beta2),
+            weight_decay=self.adam_weight_decay,
+            with_cuda=self.with_cuda,
+            cuda_devices=self.cuda_devices,
+            log_freq=self.log_freq,
+            is_logkey=self.is_logkey,
+            is_time=self.is_time,
+            hypersphere_loss=self.hypersphere_loss
+        )
+
+        self.start_iteration(surfix_log="log2")
+        self.plot_train_valid_loss("_log2")
+
+    def start_iteration(self, surfix_log):
+        print("Training Start")
+        best_loss = float('inf')
+        epochs_no_improve = 0
+
+        for epoch in range(self.epochs):
+            print("\n")
+            if self.hypersphere_loss:
+                center = self.calculate_center([self.train_data_loader, self.valid_data_loader])
+                self.trainer.hyper_center = center
+
+            _, train_dist = self.trainer.train(epoch)
+            avg_loss, valid_dist = self.trainer.valid(epoch)
+
+            self.trainer.save_log(self.model_dir, surfix_log)
+
+            if self.hypersphere_loss:
+                self.trainer.radius = self.trainer.get_radius(train_dist + valid_dist, self.trainer.nu)
+
+            if avg_loss < best_loss:
+                best_loss = avg_loss
+                self.trainer.save(self.model_path)
+                epochs_no_improve = 0
+
+                if epoch > 10 and self.hypersphere_loss:
+                    best_center = self.trainer.hyper_center
+                    best_radius = self.trainer.radius
+                    total_dist = train_dist + valid_dist
+
+                    if best_center is None:
+                        raise TypeError("center is None")
+
+                    print("best radius", best_radius)
+
+                    best_center_path = self.model_dir + "best_center.pt"
+                    print("Save best center", best_center_path)
+                    torch.save({"center": best_center, "radius": best_radius}, best_center_path)
+
+                    total_dist_path = self.model_dir + "best_total_dist.pt"
+                    print("save total dist: ", total_dist_path)
+                    torch.save(total_dist, total_dist_path)
+            else:
+                epochs_no_improve += 1
+
+            if epochs_no_improve == self.n_epochs_stop:
+                print("Early stopping")
+                break
+
+    def calculate_center(self, data_loader_list):
+        print("start calculate center")
+        with torch.no_grad():
+            outputs = 0
+            total_samples = 0
+            for data_loader in data_loader_list:
+                totol_length = len(data_loader)
+                data_iter = tqdm.tqdm(enumerate(data_loader), total=totol_length)
+                for i, data in data_iter:
+                    data = {key: value.to(self.device) for key, value in data.items()}
+                    result = self.trainer.model.forward(data["bert_input"], data["time_input"])
+                    cls_output = result["cls_output"]
+                    outputs += torch.sum(cls_output.detach().clone(), dim=0)
+                    total_samples += cls_output.size(0)
+        center = outputs / total_samples
+        return center
+
+    def plot_train_valid_loss(self, surfix_log):
+        train_loss = pd.read_csv(self.model_dir + f"train{surfix_log}.csv")
+        valid_loss = pd.read_csv(self.model_dir + f"valid{surfix_log}.csv")
+
+        sns.lineplot(x="epoch", y="loss", data=train_loss, label="train loss")
+        sns.lineplot(x="epoch", y="loss", data=valid_loss, label="valid loss")
+        plt.title("epoch vs train loss vs valid loss")
+        plt.legend()
+        plt.savefig(self.model_dir + "train_valid_loss.png")
+        plt.show()
+        print("plot done")

logbert_rca_pipeline_api.py

@@ -0,0 +1,209 @@
+import os
+import sys
+import re
+import ast
+import json
+import time
+import torch
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+from collections import defaultdict
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from torch.utils.data import DataLoader
+
+sys.path.append('../')
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from logparser import Drain
+from bert_pytorch.dataset import LogDataset, WordVocab
+from bert_pytorch.model.bert import BERT
+from bert_pytorch.model.log_model import BERTLog
+
+# === Constants ===
+TOP_EVENTS = 5
+MAX_RCA_TOKENS = 200
+MISTRAL_MODEL = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+# HF_CACHE = "/content/drive/MyDrive/hf_cache"
+
+# === Log Parsing ===
+def parse_log_with_drain(log_file, input_dir, output_dir):
+    regex = [
+        r"appattempt_\d+_\d+_\d+",
+        r"job_\d+_\d+",
+        r"task_\d+_\d+_[a-z]+_\d+",
+        r"container_\d+",
+        r"\b(?:\d{1,3}\.){3}\d{1,3}\b",
+        r"(?<!\w)\d{5,}(?!\w)",
+        r"[a-f0-9]{8,}"
+    ]
+    log_format = r'\[<AppId>] <Date> <Time> <Level> \[<Process>] <Component>: <Content>'
+    parser = Drain.LogParser(log_format, indir=input_dir, outdir=output_dir, depth=5, st=0.5, rex=regex, keep_para=True)
+    parser.parse(log_file)
+
+def hadoop_sampling(structured_log_path, sequence_output_path):
+    df = pd.read_csv(structured_log_path)
+    data_dict = defaultdict(list)
+    for _, row in tqdm(df.iterrows(), total=len(df), desc="🔍 Grouping logs by AppId"):
+        app_id = row.get("AppId")
+        event_id = row.get("EventId")
+        if pd.notnull(app_id) and pd.notnull(event_id):
+            data_dict[app_id].append(str(event_id))
+    pd.DataFrame(list(data_dict.items()), columns=['AppId', 'EventSequence']).to_csv(sequence_output_path, index=False)
+
+# === Utility Functions ===
+def load_parameters(param_path):
+    options = {}
+    with open(param_path, 'r') as f:
+        for line in f:
+            if ':' not in line: continue
+            key, val = line.strip().split(':', 1)
+            key, val = key.strip(), val.strip()
+            if val.lower() in ['true', 'false', 'none']:
+                val = eval(val.capitalize())
+            else:
+                try: val = int(val)
+                except ValueError:
+                    try: val = float(val)
+                    except ValueError: pass
+            options[key] = val
+    return options
+
+def load_logbert_model(options, vocab):
+    try:
+        return torch.load(options["model_path"], map_location=options["device"])
+    except:
+        bert = BERT(len(vocab), options["hidden"], options["layers"], options["attn_heads"], options["max_len"])
+        model = BERTLog(bert, vocab_size=len(vocab)).to(options["device"])
+        model.load_state_dict(torch.load(options["model_path"], map_location=options["device"]))
+        return model
+
+def load_center(path, device):
+    center = torch.load(path, map_location=device)
+    return center["center"] if isinstance(center, dict) else center
+
+def extract_sequences(path, min_len):
+    df = pd.read_csv(path)
+    data, app_ids = [], []
+    for _, row in df.iterrows():
+        try:
+            seq = ast.literal_eval(row["EventSequence"])
+            if len(seq) >= min_len:
+                data.append(seq)
+                app_ids.append(row["AppId"])
+        except:
+            continue
+    return data, app_ids
+
+def prepare_dataloader(sequences, vocab, options):
+    dummy_times = [[0] * len(seq) for seq in sequences]
+    dataset = LogDataset(sequences, dummy_times, vocab, seq_len=options["seq_len"], on_memory=True, mask_ratio=options["mask_ratio"])
+    return DataLoader(dataset, batch_size=1, shuffle=False, collate_fn=dataset.collate_fn)
+
+def calculate_mean_std(loader, model, center, device):
+    scores = []
+    with torch.no_grad():
+        for batch in tqdm(loader, desc="📏 Computing train distances..."):
+            batch = {k: v.to(device) for k, v in batch.items()}
+            cls_output = model(batch["bert_input"], batch["time_input"])["cls_output"]
+            scores.append(torch.norm(cls_output - center, dim=1).item())
+    return np.mean(scores), np.std(scores)
+
+def generate_prompt(event_templates):
+    prompt = "The system encountered a failure. Below are the key log events preceding the anomaly:\n\n"
+    for i, event in enumerate(event_templates, 1):
+        prompt += f"{i}. {event.strip()}\n"
+    prompt += "\nBased on the above log events, identify the most likely root cause of the issue.\n"
+    prompt += "Explain the cause in one or two sentences, using technical reasoning if possible.\n"
+    return prompt
+
+def call_mistral(prompt, tokenizer, model, device):
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
+    outputs = model.generate(
+        **inputs,
+        max_length=inputs['input_ids'].shape[1] + MAX_RCA_TOKENS,
+        do_sample=False,
+        top_k=50,
+        pad_token_id=tokenizer.eos_token_id
+    )
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)[len(prompt):].strip()
+
+def compute_logkey_anomaly(masked_output, masked_label, top_k=5):
+    num_undetected = 0
+    for i, token in enumerate(masked_label):
+        if token not in torch.argsort(-masked_output[i])[:top_k]:
+            num_undetected += 1
+    return num_undetected, len(masked_label)
+
+# === API-Compatible RCA Pipeline ===
+def detect_anomalies_and_explain(input_log_path):
+    log_file = os.path.basename(input_log_path)
+    input_dir = os.path.dirname(input_log_path)
+    output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "model", "bert"))
+
+    log_structured_file = os.path.join(output_dir, log_file + "_structured.csv")
+    log_templates_file = os.path.join(output_dir, log_file + "_templates.csv")
+    log_sequence_file = os.path.join(output_dir, "rca_abnormal_sequence.csv")
+    PARAMS_FILE = os.path.join(output_dir, "bert", "parameters.txt")
+    CENTER_PATH = os.path.join(output_dir, "bert", "best_center.pt")
+    TRAIN_FILE = os.path.join(output_dir, "train")
+
+    # Step 1: Preprocess Logs
+    parse_log_with_drain(log_file, input_dir, output_dir)
+    hadoop_sampling(log_structured_file, log_sequence_file)
+
+    # Step 2: Load Models and Parameters
+    options = load_parameters(PARAMS_FILE)
+    options["device"] = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # tokenizer = AutoTokenizer.from_pretrained(MISTRAL_MODEL)
+    # model_mistral = AutoModelForCausalLM.from_pretrained(MISTRAL_MODEL, torch_dtype=torch.float32).to(options["device"])
+    # model_mistral.eval()
+
+    vocab = WordVocab.load_vocab(options["vocab_path"])
+    model = load_logbert_model(options, vocab).to(options["device"]).eval()
+    center = load_center(CENTER_PATH, options["device"])
+
+    # Step 3: Prepare Data
+    test_sequences, app_ids = extract_sequences(log_sequence_file, options["min_len"])
+    test_loader = prepare_dataloader(test_sequences, vocab, options)
+
+    train_sequences = [line.strip().split() for line in open(TRAIN_FILE) if len(line.strip().split()) >= options["min_len"]]
+    train_loader = prepare_dataloader(train_sequences, vocab, options)
+    mean, std = calculate_mean_std(train_loader, model, center, options["device"])
+
+    templates_df = pd.read_csv(log_templates_file)
+    event_template_dict = dict(zip(templates_df["EventId"], templates_df["EventTemplate"]))
+
+    # Step 4: Analyze & Explain Anomalies
+    results = []
+    for i, batch in enumerate(test_loader):
+        batch = {k: v.to(options["device"]) for k, v in batch.items()}
+        output = model(batch["bert_input"], batch["time_input"])
+        cls_output = output["cls_output"]
+        score = torch.norm(cls_output - center, dim=1).item()
+        z_score = (score - mean) / std
+
+        num_undetected, masked_total = compute_logkey_anomaly(output["logkey_output"][0], batch["bert_label"][0])
+        undetected_ratio = num_undetected / masked_total if masked_total else 0
+
+        status = "Abnormal" if z_score > 2 or undetected_ratio > 0.5 else "Normal"
+        if status == "Normal":
+            continue
+
+        top_eids = test_sequences[i][:TOP_EVENTS]
+        event_templates = [event_template_dict.get(eid, f"[Missing Event {eid}]") for eid in top_eids]
+        #prompt = ''#generate_prompt(event_templates)
+        #explanation = ''#call_mistral(prompt, tokenizer, model_mistral, options["device"])
+
+        results.append({
+            "AppId": app_ids[i],
+            "Score": score,
+            "z_score": z_score,
+            "UndetectedRatio": undetected_ratio,
+            "status":status,
+            "Events": event_templates,
+            "Explanation": None
+        })
+
+    return results

requirements.txt

@@ -0,0 +1,19 @@
+fastapi
+uvicorn
+boto3
+botocore
+redis
+python-dotenv
+python-multipart
+torch
+transformers
+tqdm
+pandas
+numpy
+scikit-learn
+databases
+sqlalchemy
+asyncpg
+logparser
+bert_pytorch
+seaborn