diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 692d7fc..afb7e61 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -4,9 +4,16 @@
-
+
-
+
+
+
+
+
+
+
+
@@ -17,6 +24,7 @@
@@ -54,19 +62,28 @@
"associatedIndex": 0
}
+
+
+
- {
- "keyToString": {
- "ModuleVcsDetector.initialDetectionPerformed": "true",
- "RunOnceActivity.ShowReadmeOnStart": "true",
- "RunOnceActivity.git.unshallow": "true",
- "git-widget-placeholder": "Hardik/Finetune"
+
+}]]>
+
+
+
+
+
@@ -151,7 +168,23 @@
1754905922944
-
+
+
+ 1757439566361
+
+
+
+ 1757439566361
+
+
+
+ 1757440531699
+
+
+
+ 1757440531700
+
+
@@ -160,17 +193,7 @@
-
-
-
-
-
-
- file://$PROJECT_DIR$/main.py
- 8
-
-
-
-
+
+
\ No newline at end of file
diff --git a/Dataset/template.py b/Dataset/template.py
deleted file mode 100644
index 9749066..0000000
--- a/Dataset/template.py
+++ /dev/null
@@ -1,7 +0,0 @@
-"""
-Save all dataset in this path with template -
-Dataset Name: /
- Raw Dataset
- Test
- Train
-"""
\ No newline at end of file
diff --git a/Processing/Dataset/base_dataset.py b/Processing/Dataset/base_dataset.py
index 7144398..a25d938 100644
--- a/Processing/Dataset/base_dataset.py
+++ b/Processing/Dataset/base_dataset.py
@@ -1,38 +1,95 @@
-from abc import ABC, abstractmethod
-
-class BaseDataset(ABC):
- """
- Base class for datasets.
-
- Datasets should inherit from this class and implement the required methods
- for preprocessing and create required data structures for training.
- """
-
- def __init__(self,
- dataset_name: str,
- dataset_path: str,
- **kwargs):
- """
- Initialize the dataset.
-
- Args:
- dataset_name: Name of the dataset
- split: Split of the dataset
- **kwargs: Additional dataset-specific arguments
- """
- self.dataset_name = dataset_name
- self.dataset_path = dataset_path
-
- def load_csv(self):
- """
- Load dataset from a CSV file.
-
- Returns:
- Loaded dataset
- """
- import pandas as pd
- return pd.read_csv(self.dataset_path)
-
- @abstractmethod
- def format_csv(self):
- pass
+import os, json, random, requests, tarfile
+from io import BytesIO
+from google.colab import drive
+
+drive.mount('/content/drive')
+drive_dir = "/content/drive/MyDrive/adfa_finetune_data"
+os.makedirs(drive_dir, exist_ok=True)
+print(f"š¹ All datasets will be saved in: {drive_dir}")
+
+datasets = {
+ "adfa": {
+ "normal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/adfa_verazuo/adfa_test_normal",
+ "abnormal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/adfa_verazuo/adfa_test_abnormal"
+ },
+ "hdfs": {
+ "normal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/hdfs_loghub/hdfs_test_normal",
+ "abnormal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/hdfs_loghub/hdfs_test_abnormal"
+ },
+ "openstack": {
+ "normal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/openstack_loghub/openstack_test_normal",
+ "abnormal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/openstack_loghub/openstack_test_abnormal"
+ },
+ "thunderbird": {
+ "normal": "https://github.com/ait-aecid/anomaly-detection-log-datasets/raw/main/thunderbird_cfdr/thunderbird_test_normal.tar.gz",
+ "abnormal": "https://github.com/ait-aecid/anomaly-detection-log-datasets/raw/main/thunderbird_cfdr/thunderbird_test_abnormal.tar.gz"
+ }
+}
+
+def read_plain(url):
+ print(f"š„ Downloading {url}")
+ r = requests.get(url)
+ r.raise_for_status()
+ return [l.strip() for l in r.text.splitlines() if l.strip()]
+
+def read_tar(url):
+ print(f"š¦ Extracting {url}")
+ r = requests.get(url)
+ r.raise_for_status()
+ lines=[]
+ with tarfile.open(fileobj=BytesIO(r.content), mode="r:gz") as tar:
+ for m in tar.getmembers():
+ if m.isfile():
+ f = tar.extractfile(m)
+ if f:
+ txt = f.read().decode(errors="ignore")
+ lines += [l.strip() for l in txt.splitlines() if l.strip()]
+ return lines
+
+os.makedirs("data", exist_ok=True)
+
+for name, urls in datasets.items():
+ print(f"\nš¹ Processing dataset: {name}")
+ if name == "thunderbird":
+ normals = read_tar(urls["normal"])
+ abnormals = read_tar(urls["abnormal"])
+ else:
+ normals = read_plain(urls["normal"])
+ abnormals = read_plain(urls["abnormal"])
+
+ all_items = (
+ [{"dataset": name, "text": t, "label": "normal"} for t in normals] +
+ [{"dataset": name, "text": t, "label": "abnormal"} for t in abnormals]
+ )
+ random.shuffle(all_items)
+
+ split = int(0.8 * len(all_items))
+ train = all_items[:split]
+ test = all_items[split:]
+
+ def fmt(e):
+ return {
+ "instruction": "Classify the following system log as normal or abnormal.",
+ "input": f"[{e['dataset'].upper()}] {e['text']}",
+ "output": e["label"]
+ }
+
+ train_fmt = [fmt(e) for e in train]
+ test_fmt = [fmt(e) for e in test]
+
+ local_train = f"data/train_{name}.json"
+ local_test = f"data/test_{name}.json"
+ drive_train = os.path.join(drive_dir, f"train_{name}.json")
+ drive_test = os.path.join(drive_dir, f"test_{name}.json")
+
+ for path, data in [(local_train, train_fmt), (local_test, test_fmt),
+ (drive_train, train_fmt), (drive_test, test_fmt)]:
+ with open(path, "w", encoding="utf-8") as f:
+ for o in data:
+ f.write(json.dumps(o, ensure_ascii=False) + "\n")
+
+ print(f"ā
Saved {len(train_fmt)} train + {len(test_fmt)} test entries for {name}")
+ print(f" ā³ {drive_train}")
+ print(f" ā³ {drive_test}")
+
+print("\nšÆ All datasets processed and saved to Drive!")
\ No newline at end of file
diff --git a/Processing/finetuning/adfs_qwen2.py b/Processing/finetuning/adfs_qwen2.py
index 22dbbec..19dcffe 100644
--- a/Processing/finetuning/adfs_qwen2.py
+++ b/Processing/finetuning/adfs_qwen2.py
@@ -1,24 +1,115 @@
-from base_finetuning import BaseFineTune
-import logging
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
-LOCAL_MODEL_PATH = "models/adfa_qwen2"
-
-class ADFAQwen2FineTune(BaseFineTune):
- def __init__(self):
- self.model_name = MODEL_NAME
- super().__init__(self.model_name,logger=logger)
-
- def finetune(self):
- # Implement fine-tuning logic for ADFA Qwen2 model
- pass
-
- def save_checkpoint(self):
- # Implement saving checkpoint logic for ADFA Qwen2 model
- pass
-
- def upload_to_hf(self):
- # Implement uploading to Hugging Face logic for ADFA Qwen2 model
- pass
\ No newline at end of file
+import os
+import torch
+from datasets import load_dataset
+from transformers import (
+ AutoTokenizer,
+ AutoModelForCausalLM,
+ TrainingArguments,
+ Trainer,
+)
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from transformers import BitsAndBytesConfig
+
+train_path = "/content/data/train_adfa.json"
+test_path = "/content/data/test_adfa.json"
+
+dataset_name = os.path.splitext(os.path.basename(train_path))[0].replace("train_", "")
+save_dir = f"./qwen_lora_{dataset_name}_model"
+
+print("š§ Loading datasets...")
+dataset = load_dataset(
+ "json",
+ data_files={"train": train_path, "validation": test_path},
+)
+
+print(dataset)
+
+def build_prompt(instruction, inp, output):
+ """Format each sample into an instruction-style prompt."""
+ return (
+ f"Instruction: {instruction}\n"
+ f"Input: {inp}\n"
+ f"Response: {output}"
+ )
+
+print("š” Loading tokenizer...")
+model_name = "Qwen/Qwen2-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+tokenizer.pad_token = tokenizer.eos_token
+
+bnb_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_quant_type="nf4",
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+print("āļø Loading model with 4-bit quantization...")
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ quantization_config=bnb_config,
+ device_map="auto",
+)
+model = prepare_model_for_kbit_training(model)
+
+print("šŖ¶ Applying LoRA configuration...")
+lora_config = LoraConfig(
+ r=64,
+ lora_alpha=16,
+ target_modules=["q_proj", "v_proj"],
+ lora_dropout=0.05,
+ bias="none",
+ task_type="CAUSAL_LM",
+)
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+
+print("āļø Tokenizing dataset...")
+
+def tokenize_function(batch):
+ texts = [
+ build_prompt(inst, inp, out)
+ for inst, inp, out in zip(batch["instruction"], batch["input"], batch["output"])
+ ]
+ tokens = tokenizer(
+ texts,
+ padding="max_length",
+ truncation=True,
+ max_length=1024,
+ )
+ tokens["labels"] = tokens["input_ids"].copy()
+ return tokens
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset["train"].column_names)
+
+args = TrainingArguments(
+ output_dir=save_dir,
+ per_device_train_batch_size=1,
+ gradient_accumulation_steps=4,
+ num_train_epochs=1,
+ learning_rate=2e-4,
+ fp16=True,
+ save_total_limit=2,
+ evaluation_strategy="epoch",
+ logging_dir="./logs",
+ logging_steps=25,
+ report_to="none",
+)
+
+trainer = Trainer(
+ model=model,
+ args=args,
+ train_dataset=tokenized_dataset["train"],
+ eval_dataset=tokenized_dataset["validation"],
+ tokenizer=tokenizer,
+)
+
+print("š Starting training...")
+trainer.train()
+
+print(f"š¾ Saving LoRA fine-tuned model to: {save_dir}")
+os.makedirs(save_dir, exist_ok=True)
+model.save_pretrained(save_dir)
+tokenizer.save_pretrained(save_dir)
+
+print(f"ā
Model and tokenizer saved successfully at: {save_dir}")
\ No newline at end of file
diff --git a/Processing/finetuning/base_finetuning.py b/Processing/finetuning/base_finetuning.py
deleted file mode 100644
index ed272c0..0000000
--- a/Processing/finetuning/base_finetuning.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import logging
-import os
-from abc import abstractmethod, ABC
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-LOCAL_MODEL_PATH = "QWEN2/qwen2.5-1.5B-instruct"
-
-class BaseFineTune(ABC):
- def __int__(self,
- model_name: str,
- logger = None,
- *args, **kwargs):
- self.model_name = model_name
- self.tokenizer = None
- self.model = None
- self.logger = logger if logger else logging.getLogger(__name__)
-
- if not os.path.exists(LOCAL_MODEL_PATH):
- self.load()
-
- def load(self):
- """
- Load the ADFA Qwen2 model and tokenizer.
- """
- try:
- self.logger.info("Loading ADFA Qwen2 model...")
- self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
- self.model = AutoModelForCausalLM.from_pretrained(
- self.model_name,
- torch_dtype="auto",
- device_map="auto"
- )
- self.logger.info("Model and tokenizer loaded successfully.")
- self.logger.info(f"Saving model and tokenizer to: {self.local_model_path}")
- self.model.save_pretrained(LOCAL_MODEL_PATH)
- self.tokenizer.save_pretrained(LOCAL_MODEL_PATH)
- self.logger.info("ā
Model saved successfully to your local machine!")
- except Exception as e:
- self.logger.error(f"Error loading model {self.model_name}: {e}")
- raise e
-
- @abstractmethod
- def finetune(self):
- pass
-
- @abstractmethod
- def save_checkpoint(self):
- pass
-
- @abstractmethod
- def upload_to_hf(self):
- pass
\ No newline at end of file
diff --git a/Processing/finetuning/hdfs_qwen2.py b/Processing/finetuning/hdfs_qwen2.py
index ba624b7..bf34d66 100644
--- a/Processing/finetuning/hdfs_qwen2.py
+++ b/Processing/finetuning/hdfs_qwen2.py
@@ -1,24 +1,115 @@
-from base_finetuning import BaseFineTune
-import logging
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
-LOCAL_MODEL_PATH = "models/hdfs_qwen2"
-
-class ADFAQwen2FineTune(BaseFineTune):
- def __init__(self):
- self.model_name = MODEL_NAME
- super().__init__(self.model_name,logger=logger)
-
- def finetune(self):
- # Implement fine-tuning logic for ADFA Qwen2 model
- pass
-
- def save_checkpoint(self):
- # Implement saving checkpoint logic for ADFA Qwen2 model
- pass
-
- def upload_to_hf(self):
- # Implement uploading to Hugging Face logic for ADFA Qwen2 model
- pass
\ No newline at end of file
+import os
+import torch
+from datasets import load_dataset
+from transformers import (
+ AutoTokenizer,
+ AutoModelForCausalLM,
+ TrainingArguments,
+ Trainer,
+)
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from transformers import BitsAndBytesConfig
+
+train_path = "/content/data/train_hdfs.json"
+test_path = "/content/data/test_hdfs.json"
+
+dataset_name = os.path.splitext(os.path.basename(train_path))[0].replace("train_", "")
+save_dir = f"./qwen_lora_{dataset_name}_model"
+
+print("š§ Loading datasets...")
+dataset = load_dataset(
+ "json",
+ data_files={"train": train_path, "validation": test_path},
+)
+
+print(dataset)
+
+def build_prompt(instruction, inp, output):
+ """Format each sample into an instruction-style prompt."""
+ return (
+ f"Instruction: {instruction}\n"
+ f"Input: {inp}\n"
+ f"Response: {output}"
+ )
+
+print("š” Loading tokenizer...")
+model_name = "Qwen/Qwen2-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+tokenizer.pad_token = tokenizer.eos_token
+
+bnb_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_quant_type="nf4",
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+print("āļø Loading model with 4-bit quantization...")
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ quantization_config=bnb_config,
+ device_map="auto",
+)
+model = prepare_model_for_kbit_training(model)
+
+print("šŖ¶ Applying LoRA configuration...")
+lora_config = LoraConfig(
+ r=64,
+ lora_alpha=16,
+ target_modules=["q_proj", "v_proj"],
+ lora_dropout=0.05,
+ bias="none",
+ task_type="CAUSAL_LM",
+)
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+
+print("āļø Tokenizing dataset...")
+
+def tokenize_function(batch):
+ texts = [
+ build_prompt(inst, inp, out)
+ for inst, inp, out in zip(batch["instruction"], batch["input"], batch["output"])
+ ]
+ tokens = tokenizer(
+ texts,
+ padding="max_length",
+ truncation=True,
+ max_length=1024,
+ )
+ tokens["labels"] = tokens["input_ids"].copy()
+ return tokens
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset["train"].column_names)
+
+args = TrainingArguments(
+ output_dir=save_dir,
+ per_device_train_batch_size=1,
+ gradient_accumulation_steps=4,
+ num_train_epochs=1,
+ learning_rate=2e-4,
+ fp16=True,
+ save_total_limit=2,
+ evaluation_strategy="epoch",
+ logging_dir="./logs",
+ logging_steps=25,
+ report_to="none",
+)
+
+trainer = Trainer(
+ model=model,
+ args=args,
+ train_dataset=tokenized_dataset["train"],
+ eval_dataset=tokenized_dataset["validation"],
+ tokenizer=tokenizer,
+)
+
+print("š Starting training...")
+trainer.train()
+
+print(f"š¾ Saving LoRA fine-tuned model to: {save_dir}")
+os.makedirs(save_dir, exist_ok=True)
+model.save_pretrained(save_dir)
+tokenizer.save_pretrained(save_dir)
+
+print(f"ā
Model and tokenizer saved successfully at: {save_dir}")
\ No newline at end of file
diff --git a/Processing/finetuning/openstack_qwen2.py b/Processing/finetuning/openstack_qwen2.py
index c020603..6c59ec0 100644
--- a/Processing/finetuning/openstack_qwen2.py
+++ b/Processing/finetuning/openstack_qwen2.py
@@ -1,25 +1,115 @@
-from base_finetuning import BaseFineTune
-import logging
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
-LOCAL_MODEL_PATH = "models/openstack_qwen2"
-
-class ADFAQwen2FineTune(BaseFineTune):
- def __init__(self):
- self.model_name = MODEL_NAME
- self.local_model_path = LOCAL_MODEL_PATH
- super().__init__(self.model_name,logger=logger)
-
- def finetune(self):
- # Implement fine-tuning logic for ADFA Qwen2 model
- pass
-
- def save_checkpoint(self):
- # Implement saving checkpoint logic for ADFA Qwen2 model
- pass
-
- def upload_to_hf(self):
- # Implement uploading to Hugging Face logic for ADFA Qwen2 model
- pass
\ No newline at end of file
+import os
+import torch
+from datasets import load_dataset
+from transformers import (
+ AutoTokenizer,
+ AutoModelForCausalLM,
+ TrainingArguments,
+ Trainer,
+)
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from transformers import BitsAndBytesConfig
+
+train_path = "/content/data/train_openstack.json"
+test_path = "/content/data/test_openstack.json"
+
+dataset_name = os.path.splitext(os.path.basename(train_path))[0].replace("train_", "")
+save_dir = f"./qwen_lora_{dataset_name}_model"
+
+print("š§ Loading datasets...")
+dataset = load_dataset(
+ "json",
+ data_files={"train": train_path, "validation": test_path},
+)
+
+print(dataset)
+
+def build_prompt(instruction, inp, output):
+ """Format each sample into an instruction-style prompt."""
+ return (
+ f"Instruction: {instruction}\n"
+ f"Input: {inp}\n"
+ f"Response: {output}"
+ )
+
+print("š” Loading tokenizer...")
+model_name = "Qwen/Qwen2-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+tokenizer.pad_token = tokenizer.eos_token
+
+bnb_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_quant_type="nf4",
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+print("āļø Loading model with 4-bit quantization...")
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ quantization_config=bnb_config,
+ device_map="auto",
+)
+model = prepare_model_for_kbit_training(model)
+
+print("šŖ¶ Applying LoRA configuration...")
+lora_config = LoraConfig(
+ r=64,
+ lora_alpha=16,
+ target_modules=["q_proj", "v_proj"],
+ lora_dropout=0.05,
+ bias="none",
+ task_type="CAUSAL_LM",
+)
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+
+print("āļø Tokenizing dataset...")
+
+def tokenize_function(batch):
+ texts = [
+ build_prompt(inst, inp, out)
+ for inst, inp, out in zip(batch["instruction"], batch["input"], batch["output"])
+ ]
+ tokens = tokenizer(
+ texts,
+ padding="max_length",
+ truncation=True,
+ max_length=1024,
+ )
+ tokens["labels"] = tokens["input_ids"].copy()
+ return tokens
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset["train"].column_names)
+
+args = TrainingArguments(
+ output_dir=save_dir,
+ per_device_train_batch_size=1,
+ gradient_accumulation_steps=4,
+ num_train_epochs=1,
+ learning_rate=2e-4,
+ fp16=True,
+ save_total_limit=2,
+ evaluation_strategy="epoch",
+ logging_dir="./logs",
+ logging_steps=25,
+ report_to="none",
+)
+
+trainer = Trainer(
+ model=model,
+ args=args,
+ train_dataset=tokenized_dataset["train"],
+ eval_dataset=tokenized_dataset["validation"],
+ tokenizer=tokenizer,
+)
+
+print("š Starting training...")
+trainer.train()
+
+print(f"š¾ Saving LoRA fine-tuned model to: {save_dir}")
+os.makedirs(save_dir, exist_ok=True)
+model.save_pretrained(save_dir)
+tokenizer.save_pretrained(save_dir)
+
+print(f"ā
Model and tokenizer saved successfully at: {save_dir}")
\ No newline at end of file
diff --git a/Processing/finetuning/thunderbird_qwen2.py b/Processing/finetuning/thunderbird_qwen2.py
index 6b2683d..58288b0 100644
--- a/Processing/finetuning/thunderbird_qwen2.py
+++ b/Processing/finetuning/thunderbird_qwen2.py
@@ -1,25 +1,115 @@
-from base_finetuning import BaseFineTune
-import logging
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
-LOCAL_MODEL_PATH = "models/thunderbird_qwen2"
-
-class ADFAQwen2FineTune(BaseFineTune):
- def __init__(self):
- self.model_name = MODEL_NAME
- self.local_model_path = LOCAL_MODEL_PATH
- super().__init__(self.model_name,logger=logger)
-
- def finetune(self):
- # Implement fine-tuning logic for ADFA Qwen2 model
- pass
-
- def save_checkpoint(self):
- # Implement saving checkpoint logic for ADFA Qwen2 model
- pass
-
- def upload_to_hf(self):
- # Implement uploading to Hugging Face logic for ADFA Qwen2 model
- pass
\ No newline at end of file
+import os
+import torch
+from datasets import load_dataset
+from transformers import (
+ AutoTokenizer,
+ AutoModelForCausalLM,
+ TrainingArguments,
+ Trainer,
+)
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from transformers import BitsAndBytesConfig
+
+train_path = "/content/data/train_thunderbird.json"
+test_path = "/content/data/test_thunderbird.json"
+
+dataset_name = os.path.splitext(os.path.basename(train_path))[0].replace("train_", "")
+save_dir = f"./qwen_lora_{dataset_name}_model"
+
+print("š§ Loading datasets...")
+dataset = load_dataset(
+ "json",
+ data_files={"train": train_path, "validation": test_path},
+)
+
+print(dataset)
+
+def build_prompt(instruction, inp, output):
+ """Format each sample into an instruction-style prompt."""
+ return (
+ f"Instruction: {instruction}\n"
+ f"Input: {inp}\n"
+ f"Response: {output}"
+ )
+
+print("š” Loading tokenizer...")
+model_name = "Qwen/Qwen2-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+tokenizer.pad_token = tokenizer.eos_token
+
+bnb_config = BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_quant_type="nf4",
+ bnb_4bit_use_double_quant=True,
+ bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+print("āļø Loading model with 4-bit quantization...")
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ quantization_config=bnb_config,
+ device_map="auto",
+)
+model = prepare_model_for_kbit_training(model)
+
+print("šŖ¶ Applying LoRA configuration...")
+lora_config = LoraConfig(
+ r=64,
+ lora_alpha=16,
+ target_modules=["q_proj", "v_proj"],
+ lora_dropout=0.05,
+ bias="none",
+ task_type="CAUSAL_LM",
+)
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+
+print("āļø Tokenizing dataset...")
+
+def tokenize_function(batch):
+ texts = [
+ build_prompt(inst, inp, out)
+ for inst, inp, out in zip(batch["instruction"], batch["input"], batch["output"])
+ ]
+ tokens = tokenizer(
+ texts,
+ padding="max_length",
+ truncation=True,
+ max_length=1024,
+ )
+ tokens["labels"] = tokens["input_ids"].copy()
+ return tokens
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset["train"].column_names)
+
+args = TrainingArguments(
+ output_dir=save_dir,
+ per_device_train_batch_size=1,
+ gradient_accumulation_steps=4,
+ num_train_epochs=1,
+ learning_rate=2e-4,
+ fp16=True,
+ save_total_limit=2,
+ evaluation_strategy="epoch",
+ logging_dir="./logs",
+ logging_steps=25,
+ report_to="none",
+)
+
+trainer = Trainer(
+ model=model,
+ args=args,
+ train_dataset=tokenized_dataset["train"],
+ eval_dataset=tokenized_dataset["validation"],
+ tokenizer=tokenizer,
+)
+
+print("š Starting training...")
+trainer.train()
+
+print(f"š¾ Saving LoRA fine-tuned model to: {save_dir}")
+os.makedirs(save_dir, exist_ok=True)
+model.save_pretrained(save_dir)
+tokenizer.save_pretrained(save_dir)
+
+print(f"ā
Model and tokenizer saved successfully at: {save_dir}")
\ No newline at end of file
diff --git a/main.py b/main.py
deleted file mode 100644
index 94e3a87..0000000
--- a/main.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# This is a sample Python script.
-
-# Press āR to execute it or replace it with your code.
-# Press Double ā§ to search everywhere for classes, files, tool windows, actions, and settings.
-
-
-def print_hi(name):
- # Use a breakpoint in the code line below to debug your script.
- print(f'Hi, {name}') # Press āF8 to toggle the breakpoint.
-
-
-# Press the green button in the gutter to run the script.
-if __name__ == '__main__':
- print_hi('PyCharm')
-
-# See PyCharm help at https://www.jetbrains.com/help/pycharm/
diff --git a/src/SystemLogLLM/main.py b/src/SystemLogLLM/main.py
index 8d1fcc0..8768993 100644
--- a/src/SystemLogLLM/main.py
+++ b/src/SystemLogLLM/main.py
@@ -1,4 +1,124 @@
-"""
-This will be fastapi endpoint to use the SystemLogLLM model for inference.
-"""
+import os
+import json
+import torch
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+from sklearn.metrics import accuracy_score, classification_report
+BASE_MODEL = "Qwen/Qwen2-1.5B-Instruct"
+
+MODEL_PATHS = {
+ "adfa": "QWEN2/adfa",
+ "hdfs": "QWEN2/hdfs",
+ "openstack": "QWEN2/openstack",
+ "thunderbird": "QWEN2/thunderbird",
+}
+
+ORCHESTRATOR_PATH = "QWEN2/orchestrator"
+TEST_PATH = "data/test_adfa.json"
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+def load_lora_model(path):
+ """Load a Qwen base model and attach LoRA weights."""
+ print(f"š¹ Loading model from {path}")
+ base = AutoModelForCausalLM.from_pretrained(
+ BASE_MODEL,
+ device_map="auto",
+ torch_dtype=torch.bfloat16,
+ load_in_4bit=True,
+ )
+ model = PeftModel.from_pretrained(base, path)
+ model = model.merge_and_unload()
+ model.eval()
+ return model
+
+print("š Loading models...\n")
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
+
+orchestrator = load_lora_model(ORCHESTRATOR_PATH)
+domain_models = {name: load_lora_model(path) for name, path in MODEL_PATHS.items()}
+print(f"\nā
Loaded {len(domain_models)} domain models and orchestrator.\n")
+
+with open(TEST_PATH, "r") as f:
+ test_data = json.load(f)
+print(f"š§© Loaded {len(test_data)} test samples.\n")
+
+def get_orchestrator_decision(log_sample):
+ """Use the orchestrator to decide which log model to route to."""
+ prompt = f"""
+ You are a log classifier.
+ Decide which system this log belongs to: adfa, hdfs, openstack, thunderbird.
+
+ Log entry: {log_sample['input']}
+
+ Reply with only one system name.
+ """
+ inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
+ with torch.no_grad():
+ outputs = orchestrator.generate(
+ **inputs,
+ max_new_tokens=20,
+ temperature=0.3,
+ do_sample=False,
+ pad_token_id=tokenizer.eos_token_id,
+ )
+ text = tokenizer.decode(outputs[0], skip_special_tokens=True).lower()
+ for name in MODEL_PATHS.keys():
+ if name in text:
+ return name
+ return "adfa"
+
+def generate_response(model, log_text):
+ """Generate prediction (attack/normal or label) from domain model."""
+ prompt = f"Analyze the following log and predict the label:\n\n{log_text}\n\nPrediction:"
+ inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
+ with torch.no_grad():
+ outputs = model.generate(
+ **inputs,
+ max_new_tokens=20,
+ temperature=0.3,
+ pad_token_id=tokenizer.eos_token_id,
+ )
+ result = tokenizer.decode(outputs[0], skip_special_tokens=True)
+ return result.split("Prediction:")[-1].strip().lower()
+
+predictions, gold, routes = [], [], []
+
+print("š Starting inference on test set...\n")
+
+for sample in tqdm(test_data):
+ try:
+ system = get_orchestrator_decision(sample)
+ model = domain_models.get(system, domain_models["adfa"])
+
+ pred = generate_response(model, sample["input"])
+ predictions.append(pred)
+ gold.append(sample["output"].lower())
+ routes.append(system)
+ except Exception as e:
+ print(f"ā ļø Error processing sample: {e}")
+ predictions.append("error")
+ gold.append(sample["output"].lower())
+ routes.append("error")
+
+print("\nš Evaluation Summary:\n")
+acc = accuracy_score(gold, predictions)
+print(f"ā
Overall Accuracy: {acc:.4f}\n")
+
+print("š Detailed Classification Report:\n")
+print(classification_report(gold, predictions, zero_division=0))
+
+output_results = [
+ {"input": t["input"], "gold": g, "pred": p, "routed_to": r}
+ for t, g, p, r in zip(test_data, gold, predictions, routes)
+]
+
+os.makedirs("results", exist_ok=True)
+with open("results/inference_results.json", "w") as f:
+ json.dump(output_results, f, indent=2)
+
+print("\nš¾ Saved detailed predictions to results/inference_results.json")
+print("šÆ Done!\n")
\ No newline at end of file
diff --git a/template.py b/template.py
deleted file mode 100644
index e69de29..0000000