LSTM 模型，使用 SHAP 来解释预测

Question

我正在使用 LSTM 模型来检测网络中的异常情况，即 DDOS 攻击。 我已经在数据集上训练了模型。该模型正在运行，我从中得到了正确的答案。但我无法得到 SHAP 的解释。

模型训练：

#Change features with object or string value into numeric numbers
ord_feat = ['protocol_type', 'service', 'flag']
#Nom_feat = column value with 0 or 1
nom_feat = ['land', 'logged_in', 'is_host_login', 'is_guest_login']
num_feat = ['src_bytes','dst_bytes','wrong_fragment','urgent','hot','num_failed_logins','num_compromised','root_shell','su_attempted','num_root','num_file_creations','num_shells','num_access_files','num_outbound_cmds','count','srv_count','serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate','same_srv_rate','diff_srv_rate','srv_diff_host_rate','dst_host_count','dst_host_srv_count','dst_host_same_srv_rate','dst_host_diff_srv_rate','dst_host_same_src_port_rate','dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate','dst_host_rerror_rate','dst_host_srv_rerror_rate']

X_train, y_train = df.drop(columns=['class'], axis=1, inplace=False), df['class'].values


ohe = OneHotEncoder(sparse=False)
oe = OrdinalEncoder()

ohe.fit(X_train[nom_feat].values)
oe.fit(X_train[ord_feat].values)


scalar = StandardScaler()
scalar.fit(X_train[num_feat].values)
X_train_nom = ohe.transform(X_train[nom_feat].values)
X_train_ord = oe.transform(X_train[ord_feat].values)
X_train_num = scalar.transform(X_train[num_feat].values)

X_train = np.concatenate([X_train_ord, X_train_num, X_train_nom], axis=1)
#SVM Approach-------------------------------------------------------------------
from sklearn import svm

classifier = svm.SVC(kernel = "linear")
classifier.fit(X_train, y_train)
y_predict = classifier.predict(X_test)

from sklearn import metrics
print("SVM ACCURACY : ",metrics.accuracy_score(y_test, y_predict))
#SVM Approach Done--------------------------------------------------------------
#reshape Train dataset into 3d array
X_train = X_train.reshape((X_train.shape[0],1,X_train.shape[1]))
y_train = y_train.reshape((y_train.shape[0],1,1))

#Scale the num, ord, nom datasets
X_test, y_test = df_val.drop(columns=['class'], axis = 1, inplace=False), df_val['class'].values

ohe.fit(X_test[nom_feat].values)
oe.fit(X_test[ord_feat].values)
#reshape Test dataset into 3d array

scalar.fit(X_test[num_feat].values)
X_test_nom = ohe.transform(X_test[nom_feat].values)
X_test_ord = oe.transform(X_test[ord_feat].values)
X_test_num = scalar.transform(X_test[num_feat].values)
X_test = np.concatenate([X_test_ord, X_test_num, X_test_nom], axis=1)
X_test.shape
X_test = X_test.reshape((X_test.shape[0],1,X_test.shape[1]))
y_test = y_test.reshape((y_test.shape[0],1,1))
model = Sequential()
#50 time steps, and 2 features
#LSTM INOUT (Batch size, Time steps, units)
model.add(LSTM(units = 44, input_shape=(1,44), return_sequences=True))
model.add(Dense(1))
model.add(Dense(1))
model.compile(loss="mean_absolute_error", optimizer = 'adam', metrics = ["accuracy"])
# model.summary()

history = model.fit(X_train, y_train, epochs=30, validation_data = (X_test, y_test))# validation_data=(X_test, y_test)

接下来，在我使用该模型的主应用程序中，我需要得出该包为何有害的解释

初始化形状。解释器

   def initEncodersAndScaler(self):
    df = pd.read_csv("attack_test.csv")
    # Подготовка данных
    X, Y = df.drop(columns=['class'], axis = 1, inplace=False), df['class'].values

# Обучение энкодеров и масштабировщика
self.ohe.fit(X[nom_feat].values)
self.oe.fit(X[ord_feat].values)
#reshape Test dataset into 3d array
self.scalar.fit(X[num_feat].values)

X_test_nom = self.ohe.transform(X[nom_feat].values)
X_test_ord = self.oe.transform(X[ord_feat].values)
X_test_num = self.scalar.transform(X[num_feat].values)
X = np.concatenate([X_test_ord, X_test_num, X_test_nom], axis=1)    
# Создаем маскер с правильным размером
print(X.shape)
self.masker = shap.maskers.Independent(data = X)  
self.explainer = shap.Explainer(self.model, self.masker)  # Создаем объект объяснителя

这就是我使用解释器的地方：

def run(self):
    for index, row in self.data.iterrows():
        if self.stop_flag:
            break  # Если установлен флаг, прекращаем выполнение
        
        # Создаем объект с признаками пакета
        packet_features = NetworkPacketFeatures(row.to_dict())
        
        # Подготовка данных для предсказания
        features = packet_features.to_array()
        print(features.shape)
        explanation = packet_features.explainer(features)
        shap_values = explanation.values  # Получаем значения SHAP
        print(f"SHAP Values для пакета: {shap_values}")  # Выводим значения SHAP
        features = features.reshape((features.shape[0], 1, features.shape[1]))  # Подготовка данных
        prediction = packet_features.model.predict(features)  # Предсказание
        scale_pred = prediction[0][0][0]
        
        self.update_signal.emit(packet_features, scale_pred)  # Отправка сигнала

features 仅包含整个列表中的 1 个包！ 该模型还接受

之后的特征

features = features.reshape((features.shape[0], 1, features.shape[1]))  # Data preparation

但我尝试在发送给解释器之前进行重塑，但没有帮助。我遇到了数组大小不正确的错误，由于某种原因，我遇到了解释器被视为 numpy 数组的错误。 现在用我的代码，错误是这样的

ValueError: Input 0 of layer "sequential_5" is incompatible with the layer: expected shape=(None, 1, 44), found shape=(3324, 44)
Aborted (core dumped)

Answer 1

我解决了这个问题

一开始

X = np.concatenate([X_test_ord, X_test_num, X_test_nom], axis=1)
X_test_sampled = shap.sample(X, 100) # Выбираем 100 случайных образцов для фона

    # Продолжение кода с SHAP
    # Создаем SHAP-объяснитель с учетом правильной формы данных
    self.explainer = shap.KernelExplainer(
        lambda x: self.model.predict(x.reshape((x.shape[0], 1, x.shape[1]))),
        X_test_sampled # Используем правильную форму
    )

第二次

Подготовка данных для предсказания

        features = packet_features.to_array()
        features_copy = features.copy()


        # Вычисляем SHAP-значения
        shap_values = packet_features.explainer.shap_values(features_copy)

        # Убедитесь, что количество feature_names совпадает с количеством признаков в данных
        feature_names = ord_feat + num_feat + nom_feat
        num_features = shap_values[0].shape[-1]

        if len(feature_names) > num_features:
            feature_names = feature_names[:num_features]
        elif len(feature_names) < num_features:
            additional_names = [f"feature_{i}" for i in range(len(feature_names), num_features)]
            feature_names.extend(additional_names)

        # Проверьте, что размеры SHAP-значений совпадают с размерами данных
        assert shap_values[0].shape[-1] == features_copy.shape[1], "Mismatch in SHAP values shape and data shape"

       # Преобразование SHAP-значений в DataFrame для использования в QT
        shap_df = pd.DataFrame(shap_values[0], columns=feature_names)
        threshold = 0.1  # Выберите подходящий порог
        # Вывод значимых признаков (только тех, что выше порога)
        significant_features = shap_df.loc[:, (shap_df.abs() > threshold).any(axis=0)]