preprocessing.py

import os
import cv2
import numpy as np
import joblib
from clearml import Task, Dataset
from sklearn.utils import shuffle
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

def prepare_dataset(
    dataset_id: str,
    width: int = 64,
    height: int = 64,
    test_ratio: float = 0.3
):
    """
    Downloads the ClearML dataset by ID, loads/resizes images,
    encodes labels, splits into train/test, and saves artifacts.
    Returns the .npy split filenames plus label encoder path.
    """
    task = Task.init(
        project_name="Plant_Diseases_Detection",
        task_name="preprocessing",
        task_type=Task.TaskTypes.data_processing,
        output_uri="https://files.clear.ml"
    )
    task.connect({
        "dataset_id": dataset_id,
        "width": width,
        "height": height,
        "test_ratio": test_ratio
    })
    task.execute_remotely(queue_name="plant")

    # 1) Get the combined dataset
    ds_folder = Dataset.get(dataset_id=dataset_id).get_local_copy()

    # 2) Load & resize images
    images, labels = [], []
    for cls in os.listdir(ds_folder):
        cls_p = os.path.join(ds_folder, cls)
        if not os.path.isdir(cls_p):
            continue
        for f in os.listdir(cls_p):
            if f.lower().endswith((".jpg", ".jpeg", ".png")):
                img = cv2.imread(os.path.join(cls_p, f))
                img = cv2.resize(img, (width, height))
                images.append(img)
                labels.append(cls)

    images = np.array(images)
    labels = np.array(labels)
    images, labels = shuffle(images, labels, random_state=42)

    # 3) Encode labels
    le = LabelEncoder()
    y = le.fit_transform(labels)
    encoder_path = "label_encoder.joblib"
    joblib.dump(le, encoder_path)
    task.upload_artifact("label_encoder", encoder_path)

    # 4) Split
    X_train, X_test, y_train, y_test = train_test_split(
        images, y,
        test_size=test_ratio,
        random_state=42,
        stratify=y
    )

    # 5) Save splits as artifacts
    np.save("X_train.npy", X_train)
    np.save("X_test.npy",  X_test)
    np.save("y_train.npy", y_train)
    np.save("y_test.npy",  y_test)
    task.upload_artifact("X_train", "X_train.npy")
    task.upload_artifact("X_test", "X_test.npy")
    task.upload_artifact("y_train", "y_train.npy")
    task.upload_artifact("y_test", "y_test.npy")

    # Report status
    task.get_logger().report_text("Artifacts uploaded: X_train, X_test, y_train, y_test, label_encoder.")

    return "X_train.npy", "X_test.npy", "y_train.npy", "y_test.npy", encoder_path

if __name__ == "__main__":
    # Always run remotely in 'plant' queue
    DATASET_ID = "d7939308d6254bbaa20fb459beea3b63"
    outputs = prepare_dataset(DATASET_ID)
    print("Outputs:", outputs)