Hardik/Finetune - Models and finetuning fixed

Processing/Dataset/base_dataset.py

@@ -1,38 +1,95 @@
-from abc import ABC, abstractmethod
-
-class BaseDataset(ABC):
-    """
-    Base class for datasets.
-
-    Datasets should inherit from this class and implement the required methods
-    for preprocessing and create required data structures for training.
-    """
-
-    def __init__(self,
-                 dataset_name: str,
-                 dataset_path: str,
-                 **kwargs):
-        """
-        Initialize the dataset.
-
-        Args:
-            dataset_name: Name of the dataset
-            split: Split of the dataset
-            **kwargs: Additional dataset-specific arguments
-        """
-        self.dataset_name = dataset_name
-        self.dataset_path = dataset_path
-
-    def load_csv(self):
-        """
-        Load dataset from a CSV file.
-
-        Returns:
-            Loaded dataset
-        """
-        import pandas as pd
-        return pd.read_csv(self.dataset_path)
-
-    @abstractmethod
-    def format_csv(self):
-        pass
+import os, json, random, requests, tarfile
+from io import BytesIO
+from google.colab import drive
+
+drive.mount('/content/drive')
+drive_dir = "/content/drive/MyDrive/adfa_finetune_data"
+os.makedirs(drive_dir, exist_ok=True)
+print(f"🔹 All datasets will be saved in: {drive_dir}")
+
+datasets = {
+    "adfa": {
+        "normal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/adfa_verazuo/adfa_test_normal",
+        "abnormal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/adfa_verazuo/adfa_test_abnormal"
+    },
+    "hdfs": {
+        "normal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/hdfs_loghub/hdfs_test_normal",
+        "abnormal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/hdfs_loghub/hdfs_test_abnormal"
+    },
+    "openstack": {
+        "normal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/openstack_loghub/openstack_test_normal",
+        "abnormal": "https://raw.githubusercontent.com/ait-aecid/anomaly-detection-log-datasets/refs/heads/main/openstack_loghub/openstack_test_abnormal"
+    },
+    "thunderbird": {
+        "normal": "https://github.com/ait-aecid/anomaly-detection-log-datasets/raw/main/thunderbird_cfdr/thunderbird_test_normal.tar.gz",
+        "abnormal": "https://github.com/ait-aecid/anomaly-detection-log-datasets/raw/main/thunderbird_cfdr/thunderbird_test_abnormal.tar.gz"
+    }
+}
+
+def read_plain(url):
+    print(f"📥 Downloading {url}")
+    r = requests.get(url)
+    r.raise_for_status()
+    return [l.strip() for l in r.text.splitlines() if l.strip()]
+
+def read_tar(url):
+    print(f"📦 Extracting {url}")
+    r = requests.get(url)
+    r.raise_for_status()
+    lines=[]
+    with tarfile.open(fileobj=BytesIO(r.content), mode="r:gz") as tar:
+        for m in tar.getmembers():
+            if m.isfile():
+                f = tar.extractfile(m)
+                if f:
+                    txt = f.read().decode(errors="ignore")
+                    lines += [l.strip() for l in txt.splitlines() if l.strip()]
+    return lines
+
+os.makedirs("data", exist_ok=True)
+
+for name, urls in datasets.items():
+    print(f"\n🔹 Processing dataset: {name}")
+    if name == "thunderbird":
+        normals = read_tar(urls["normal"])
+        abnormals = read_tar(urls["abnormal"])
+    else:
+        normals = read_plain(urls["normal"])
+        abnormals = read_plain(urls["abnormal"])
+
+    all_items = (
+        [{"dataset": name, "text": t, "label": "normal"} for t in normals] +
+        [{"dataset": name, "text": t, "label": "abnormal"} for t in abnormals]
+    )
+    random.shuffle(all_items)
+
+    split = int(0.8 * len(all_items))
+    train = all_items[:split]
+    test  = all_items[split:]
+
+    def fmt(e):
+        return {
+            "instruction": "Classify the following system log as normal or abnormal.",
+            "input": f"[{e['dataset'].upper()}] {e['text']}",
+            "output": e["label"]
+        }
+
+    train_fmt = [fmt(e) for e in train]
+    test_fmt  = [fmt(e) for e in test]
+
+    local_train = f"data/train_{name}.json"
+    local_test  = f"data/test_{name}.json"
+    drive_train = os.path.join(drive_dir, f"train_{name}.json")
+    drive_test  = os.path.join(drive_dir, f"test_{name}.json")
+
+    for path, data in [(local_train, train_fmt), (local_test, test_fmt),
+                       (drive_train, train_fmt), (drive_test, test_fmt)]:
+        with open(path, "w", encoding="utf-8") as f:
+            for o in data:
+                f.write(json.dumps(o, ensure_ascii=False) + "\n")
+
+    print(f"✅ Saved {len(train_fmt)} train + {len(test_fmt)} test entries for {name}")
+    print(f"   ↳ {drive_train}")
+    print(f"   ↳ {drive_test}")
+
+print("\n🎯 All datasets processed and saved to Drive!")

Processing/finetuning/adfs_qwen2.py

@@ -1,24 +1,115 @@
-from base_finetuning import BaseFineTune
-import logging
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
-LOCAL_MODEL_PATH = "models/adfa_qwen2"
-
-class ADFAQwen2FineTune(BaseFineTune):
-    def __init__(self):
-        self.model_name = MODEL_NAME
-        super().__init__(self.model_name,logger=logger)
-
-    def finetune(self):
-        # Implement fine-tuning logic for ADFA Qwen2 model
-        pass
-
-    def save_checkpoint(self):
-        # Implement saving checkpoint logic for ADFA Qwen2 model
-        pass
-
-    def upload_to_hf(self):
-        # Implement uploading to Hugging Face logic for ADFA Qwen2 model
-        pass
+import os
+import torch
+from datasets import load_dataset
+from transformers import (
+    AutoTokenizer,
+    AutoModelForCausalLM,
+    TrainingArguments,
+    Trainer,
+)
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from transformers import BitsAndBytesConfig
+
+train_path = "/content/data/train_adfa.json"
+test_path = "/content/data/test_adfa.json"
+
+dataset_name = os.path.splitext(os.path.basename(train_path))[0].replace("train_", "")
+save_dir = f"./qwen_lora_{dataset_name}_model"
+
+print("🧠 Loading datasets...")
+dataset = load_dataset(
+    "json",
+    data_files={"train": train_path, "validation": test_path},
+)
+
+print(dataset)
+
+def build_prompt(instruction, inp, output):
+    """Format each sample into an instruction-style prompt."""
+    return (
+        f"Instruction: {instruction}\n"
+        f"Input: {inp}\n"
+        f"Response: {output}"
+    )
+
+print("🔡 Loading tokenizer...")
+model_name = "Qwen/Qwen2-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+tokenizer.pad_token = tokenizer.eos_token
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+print("⚙️ Loading model with 4-bit quantization...")
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    quantization_config=bnb_config,
+    device_map="auto",
+)
+model = prepare_model_for_kbit_training(model)
+
+print("🪶 Applying LoRA configuration...")
+lora_config = LoraConfig(
+    r=64,
+    lora_alpha=16,
+    target_modules=["q_proj", "v_proj"],
+    lora_dropout=0.05,
+    bias="none",
+    task_type="CAUSAL_LM",
+)
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
+
+print("✂️ Tokenizing dataset...")
+
+def tokenize_function(batch):
+    texts = [
+        build_prompt(inst, inp, out)
+        for inst, inp, out in zip(batch["instruction"], batch["input"], batch["output"])
+    ]
+    tokens = tokenizer(
+        texts,
+        padding="max_length",
+        truncation=True,
+        max_length=1024,
+    )
+    tokens["labels"] = tokens["input_ids"].copy()
+    return tokens
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset["train"].column_names)
+
+args = TrainingArguments(
+    output_dir=save_dir,
+    per_device_train_batch_size=1,
+    gradient_accumulation_steps=4,
+    num_train_epochs=1,
+    learning_rate=2e-4,
+    fp16=True,
+    save_total_limit=2,
+    evaluation_strategy="epoch",
+    logging_dir="./logs",
+    logging_steps=25,
+    report_to="none",
+)
+
+trainer = Trainer(
+    model=model,
+    args=args,
+    train_dataset=tokenized_dataset["train"],
+    eval_dataset=tokenized_dataset["validation"],
+    tokenizer=tokenizer,
+)
+
+print("🚀 Starting training...")
+trainer.train()
+
+print(f"💾 Saving LoRA fine-tuned model to: {save_dir}")
+os.makedirs(save_dir, exist_ok=True)
+model.save_pretrained(save_dir)
+tokenizer.save_pretrained(save_dir)
+
+print(f"✅ Model and tokenizer saved successfully at: {save_dir}")