preprocessing.py

import ipaddress
import os

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split


def is_internal(ip):
    try:
        return int(ipaddress.ip_address(ip).is_private)
    except ValueError:
        return 0  # IP non valido → trattalo come esterno


def data_split(df):
    # Split stratificato
    df_train, df_test = train_test_split(
        df, test_size=0.2, stratify=df["Label"], random_state=42
    )
    df_train["train_mask"] = 1
    df_test["train_mask"] = 0

    df_all = pd.concat([df_train, df_test]).reset_index(drop=True)
    return df_all


if __name__ == "__main__":
    # stampo posizione di lavoro attuale
    print(os.getcwd())

    try:
        df = pd.read_csv("data/full/UNSW-NB15_processed.csv")
    except FileNotFoundError:
        df = pd.DataFrame()

    if df.empty:
        # Leggi il file delle feature
        features = pd.read_csv("./data/full/NUSW-NB15_features.csv", encoding="latin1")
        feature_names = features["Name"].tolist()

        # Lista dei file da leggere
        csv_files = [
            "./data/full/UNSW-NB15_1.csv",
            "./data/full/UNSW-NB15_2.csv",
            "./data/full/UNSW-NB15_3.csv",
            "./data/full/UNSW-NB15_4.csv",
        ]

        # Leggi e concatena i file
        dfs = []
        for file in csv_files:
            df = pd.read_csv(file, header=None, names=feature_names)
            dfs.append(df)

        # Unisci tutto in un unico DataFrame
        df = pd.concat(dfs, ignore_index=True)

        # sostiuisco i valori NaN della colonna attack_cat con 'Normal' quando la colonna label è 0
        df["attack_cat"] = df.apply(
            lambda x: "Normal" if x["Label"] == 0 else x["attack_cat"], axis=1
        )

        df.replace(" ", np.nan, inplace=True)
        missing_percentage = df.isnull().mean() * 100
        print(missing_percentage.sort_values(ascending=False))

        print(df["ct_flw_http_mthd"].value_counts(dropna=False))
        print(df["is_ftp_login"].value_counts(dropna=False))
        df["ct_ftp_cmd"] = pd.to_numeric(df["ct_ftp_cmd"], errors="coerce")
        print(df["ct_ftp_cmd"].value_counts(dropna=False))

        # rimuovo le colonne con più del 50% di valori nulli
        df = df.drop(["ct_flw_http_mthd", "is_ftp_login", "ct_ftp_cmd"], axis=1)

        # creo le features derivate TotBytes(sbytes+dbytes), BytesPerPkt(TotBytes/spkts+dpkts), PktPerSec(spkts+dpkts/duration) e RatioOutIn(sbytes/dbytes)
        # Se RatioOutIn o PktsPerSec risultano ∞ (perché InBytes o Duration sono 0), sostituisci con il massimo valore finito della colonna.

        df["BytesPerPkt"] = (df["sbytes"] + df["dbytes"]) / (df["Spkts"] + df["Dpkts"])

        df["PktPerSec"] = (df["Spkts"] + df["Dpkts"]) / df["dur"]
        df["RatioOutIn"] = df["sbytes"] / df["dbytes"]

        df["PktPerSec"] = df["PktPerSec"].replace([np.inf, -np.inf], np.nan)
        df["RatioOutIn"] = df["RatioOutIn"].replace([np.inf, -np.inf], np.nan)
        df["PktPerSec"].fillna(df["PktPerSec"].max(), inplace=True)
        df["RatioOutIn"].fillna(df["RatioOutIn"].max(), inplace=True)

        # Rimuovi le righe che superano la soglia basata sul 95° percentile per le colonne dur, spkts, dpkts, sbytes, dbytes, Sload, Dload, PktPerSec

        threshold_dur = df["dur"].quantile(0.95)
        threshold_spkts = df["Spkts"].quantile(0.95)
        threshold_dpkts = df["Dpkts"].quantile(0.95)
        threshold_sbytes = df["sbytes"].quantile(0.95)
        threshold_dbytes = df["dbytes"].quantile(0.95)
        threshold_Sload = df["Sload"].quantile(0.95)
        threshold_Dload = df["Dload"].quantile(0.95)
        threshold_PktPerSec = df["PktPerSec"].quantile(0.95)

        df = df[df["dur"] < threshold_dur]
        df = df[df["Spkts"] < threshold_spkts]
        df = df[df["Dpkts"] < threshold_dpkts]
        df = df[df["sbytes"] < threshold_sbytes]
        df = df[df["dbytes"] < threshold_dbytes]
        df = df[df["Sload"] < threshold_Sload]
        df = df[df["Dload"] < threshold_Dload]
        df = df[df["PktPerSec"] < threshold_PktPerSec]

        # trasformo gli ip in 0 e 1 a seconda che siano interni o esterni
        df["IPSrcType"] = df["srcip"].apply(is_internal)
        df["IPDstType"] = df["dstip"].apply(is_internal)

        # Converto sport e dsport a numeri (eventuali NaN diventano -1 per coerenza)
        df["sport"] = pd.to_numeric(df["sport"], errors="coerce").fillna(-1).astype(int)
        df["dsport"] = (
            pd.to_numeric(df["dsport"], errors="coerce").fillna(-1).astype(int)
        )

        # Source Port
        df["SrcPortWellKnown"] = ((df["sport"] >= 0) & (df["sport"] <= 1023)).astype(
            int
        )
        df["SrcPortRegistered"] = (
            (df["sport"] >= 1024) & (df["sport"] <= 49151)
        ).astype(int)
        df["SrcPortPrivate"] = (df["sport"] > 49151).astype(int)

        # Destination Port
        df["DstPortWellKnown"] = ((df["dsport"] >= 0) & (df["dsport"] <= 1023)).astype(
            int
        )
        df["DstPortRegistered"] = (
            (df["dsport"] >= 1024) & (df["dsport"] <= 49151)
        ).astype(int)
        df["DstPortPrivate"] = (df["dsport"] > 49151).astype(int)

        benigni = len(df[df["Label"] == 0])
        malevoli = len(df[df["Label"] == 1])

        rapporto = benigni / malevoli
        print(f"Benigni: {benigni}, Malevoli: {malevoli}, Rapporto: {rapporto:.2f}:1")

        df.to_csv("./data/full/UNSW-NB15_processed.csv", index=False)
        print(f"Dataset completo creato con {len(df)} campioni")
    
    # creo la cartella per i sottodataset se non esiste
    if not os.path.exists("./data/attack_cat/"):
        os.makedirs("./data/attack_cat/")

    # Step 1: rimuovi spazi iniziali/finali
    df["attack_cat"] = df["attack_cat"].str.strip()

    # Step 2: uniforma a lowercase (opzionale ma consigliato per evitare duplicati)
    df["attack_cat"] = df["attack_cat"].str.lower()

    # Step 3: correggi manualmente categorie duplicate o sbagliate (opzionale ma utile)
    df["attack_cat"] = df["attack_cat"].replace(
        {
            "backdoors": "backdoor",
            "shellcode": "shellcode",  # se ci sono doppioni puoi forzare un nome
            "fuzzers": "fuzzers",
        }
    )
    
    # creo dei sottodataset per attacchi diversi con più di 1000 campioni
    # rimuovo le righe con meno di 1000 campioni
    df = df[
        df["attack_cat"].isin(
            df["attack_cat"]
            .value_counts()[df["attack_cat"].value_counts() > 1000]
            .index
        )
    ]

    # creo un df per ogni attack_cat mantenendo un rapporto 20:1
    df_benigni = df[df["Label"] == 0]

    for attack in df["attack_cat"].unique():
        if attack != "normal":
            df_attack = df[df["attack_cat"] == attack]
            num_needed = len(df_attack) * 20

            # se i benigni disponibili non bastano, cambio il rapporto a 10:1
            if num_needed > len(df_benigni):
                num_needed = len(df_attack) * 10

            df_sampled_benigni = df_benigni.sample(
                n=num_needed, replace=False, random_state=42
            )

            df_balanced = pd.concat([df_attack, df_sampled_benigni])
            # splitto in train e test
            df_balanced = data_split(df_balanced)

            df_balanced.to_csv(
                f"./data/attack_cat/UNSW-NB15_{attack}.csv",
                index=False,
            )

            print(f"Dataset {attack} creato con {len(df_balanced)} campioni")