【Python実践】「本当に効く」変数はどれか？ アンサンブル回帰モデルにおける特徴量の選定手順

import pandas as pd
import matplotlib.pyplot as plt
import os
import seaborn as sns

def visualize_features_by_column(df, target_col, exclude_cols, save_dir="feature_plots", apply_outlier_filter=False):
    """
    df: pandas DataFrame
    target_col: 目的変数のカラム名（str）
    exclude_cols: 除外対象のカラム名（カンマ区切りのstr）
    save_dir: 出力先フォルダ（str）
    apply_outlier_filter: 3σ外れ値除外を適用するか（bool）
    """
    exclude_list = [col.strip() for col in exclude_cols.split(',')]
    feature_cols = [col for col in df.columns if col != target_col and col not in exclude_list]

    sns.set_style("whitegrid")
    sns.set(font='Meiryo')
    os.makedirs(save_dir, exist_ok=True)

    for col in feature_cols:
        plot_df = df[[col, target_col]].copy()

        if apply_outlier_filter:
            # 3σで外れ値除外（X軸）
            x_mean, x_std = plot_df[col].mean(), plot_df[col].std()
            x_mask = (plot_df[col] >= x_mean - 3 * x_std) & (plot_df[col] <= x_mean + 3 * x_std)

            # 3σで外れ値除外（Y軸）
            y_mean, y_std = plot_df[target_col].mean(), plot_df[target_col].std()
            y_mask = (plot_df[target_col] >= y_mean - 3 * y_std) & (plot_df[target_col] <= y_mean + 3 * y_std)

            # 両方の条件を満たすデータのみ使用
            plot_df = plot_df[x_mask & y_mask]

        fig, axes = plt.subplots(1, 4, figsize=(20, 5))

        sns.boxplot(y=plot_df[col], ax=axes[0], color='skyblue')
        axes[0].set_title(f'Boxplot of {col}')

        sns.histplot(plot_df[col], ax=axes[1], bins=20, kde=True, color='mediumseagreen')
        axes[1].set_title(f'Histogram of {col}')

        sns.scatterplot(x=plot_df[col], y=plot_df[target_col], ax=axes[2], color='salmon')
        axes[2].set_title(f'{col} vs {target_col}')

        sns.lineplot(x=plot_df.index, y=plot_df[col], ax=axes[3], color='gray')
        axes[3].set_title(f'Time Series of {col}')

        plt.tight_layout()
        filename = os.path.join(save_dir, f"{col}_visualization.png")
        fig.savefig(filename)
        plt.close(fig)
        print(f"Saved: {filename}")

if __name__ == "__main__":
    df =pd.read_csv(r"P:\data\sample.csv",encoding="shift-jis")
    visualize_features_by_column(df,"Sensor5","Sensor1,Sensor2,Sensor3,Sensor4",r"P:\data\png")

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
import numpy as np

sns.set(font='Meiryo', style='whitegrid')

def save_top_scatter_plots_seaborn(df, target_col, exclude_cols, save_dir="scatter_plots", top=10, corr_threshold=0.6):
    """
    df: pandas DataFrame
    target_col: 目的変数のカラム名（str）
    exclude_cols: 除外対象のカラム名（カンマ区切りのstr）
    save_dir: 出力先フォルダ（str）
    top: 上位から抽出する特徴量数（int）
    """
    exclude_list = [col.strip() for col in exclude_cols.split(',')]
    feature_cols = [col for col in df.columns if col != target_col and col not in exclude_list]

    # 相関係数を目的変数に対して算出（欠損除去後）
    correlations = df[feature_cols].corrwith(df[target_col]).sort_values(key=abs, ascending=False)
    top_features = correlations[abs(correlations) >= corr_threshold].head(top).index.tolist()

    os.makedirs(save_dir, exist_ok=True)

    for col in top_features:
        subset = df[[col, target_col]].dropna()
        total_count = len(subset)

        # 元データモデル
        x = subset[col].values.reshape(-1, 1)
        y = subset[target_col].values
        model = LinearRegression().fit(x, y)
        y_pred = model.predict(x)
        r2 = r2_score(y, y_pred)
        coef = model.coef_[0]
        intercept = model.intercept_
        corr = correlations[col]

        # 目的変数(Y)の3σ範囲
        mean_y, std_y = subset[target_col].mean(), subset[target_col].std()
        lower_y, upper_y = mean_y - 3 * std_y, mean_y + 3 * std_y
        mask_y = (subset[target_col] >= lower_y) & (subset[target_col] <= upper_y)
        subset_y_filtered = subset[mask_y]
        outliers_y = subset[~mask_y]
        ratio_y = len(outliers_y) / total_count * 100

        # 説明変数(X)の3σ範囲
        mean_x, std_x = subset[col].mean(), subset[col].std()
        lower_x, upper_x = mean_x - 3 * std_x, mean_x + 3 * std_x
        mask_x = (subset[col] >= lower_x) & (subset[col] <= upper_x)
        subset_x_filtered = subset[mask_x]
        outliers_x = subset[~mask_x]
        ratio_x = len(outliers_x) / total_count * 100

        # 両方除外
        mask_xy = mask_x & mask_y
        subset_xy_filtered = subset[mask_xy]

        # 中央（Y異常値除外）モデル
        x_y = subset_y_filtered[col].values.reshape(-1, 1)
        y_y = subset_y_filtered[target_col].values
        model_y = LinearRegression().fit(x_y, y_y)
        y_pred_y = model_y.predict(x_y)
        r2_y = r2_score(y_y, y_pred_y)
        coef_y = model_y.coef_[0]
        intercept_y = model_y.intercept_
        corr_y = subset_y_filtered[col].corr(subset_y_filtered[target_col])

        # 右（X異常値除外）モデル
        x_x = subset_x_filtered[col].values.reshape(-1, 1)
        y_x = subset_x_filtered[target_col].values
        model_x = LinearRegression().fit(x_x, y_x)
        y_pred_x = model_x.predict(x_x)
        r2_x = r2_score(y_x, y_pred_x)
        coef_x = model_x.coef_[0]
        intercept_x = model_x.intercept_
        corr_x = subset_x_filtered[col].corr(subset_x_filtered[target_col])

        # 両方除外モデル
        x_xy = subset_xy_filtered[col].values.reshape(-1, 1)
        y_xy = subset_xy_filtered[target_col].values
        model_xy = LinearRegression().fit(x_xy, y_xy)
        y_pred_xy = model_xy.predict(x_xy)
        r2_xy = r2_score(y_xy, y_pred_xy)
        coef_xy = model_xy.coef_[0]
        intercept_xy = model_xy.intercept_
        corr_xy = subset_xy_filtered[col].corr(subset_xy_filtered[target_col])

        # 描画
        fig, axes = plt.subplots(1, 4, figsize=(24, 6))

        # 左：元データ
        sns.scatterplot(x=subset[col], y=subset[target_col], color='salmon', alpha=0.5, ax=axes[0])
        sns.scatterplot(x=outliers_y[col], y=outliers_y[target_col], color='red', ax=axes[0], label='Y異常値')
        sns.scatterplot(x=outliers_x[col], y=outliers_x[target_col], color='darkred', ax=axes[0], label='X異常値')
        sns.lineplot(x=subset[col], y=y_pred, color='blue', ax=axes[0])
        axes[0].set_title(
            f"元データ\n相関: {corr:.2f}, R²: {r2:.2f}\n回帰: y={coef:.2f}x+{intercept:.2f}\n"
            f"Y異常値: {len(outliers_y)}件 ({ratio_y:.1f}%), X異常値: {len(outliers_x)}件 ({ratio_x:.1f}%)",
            fontsize=10
        )

        # 中央：Y異常値除外
        sns.scatterplot(x=subset_y_filtered[col], y=subset_y_filtered[target_col], color='green', alpha=0.5, ax=axes[1])
        sns.lineplot(x=subset_y_filtered[col], y=y_pred_y, color='blue', ax=axes[1])
        axes[1].set_title(
            f"目的変数3σ除外\n相関: {corr_y:.2f}, R²: {r2_y:.2f}\n回帰: y={coef_y:.2f}x+{intercept_y:.2f}\n"
            f"除外: {len(outliers_y)}件 ({ratio_y:.1f}%), 範囲: [{lower_y:.2f}, {upper_y:.2f}]",
            fontsize=10
        )

        # 右：X異常値除外
        sns.scatterplot(x=subset_x_filtered[col], y=subset_x_filtered[target_col], color='purple', alpha=0.5, ax=axes[2])
        sns.lineplot(x=subset_x_filtered[col], y=y_pred_x, color='blue', ax=axes[2])
        axes[2].set_title(
            f"説明変数3σ除外\n相関: {corr_x:.2f}, R²: {r2_x:.2f}\n回帰: y={coef_x:.2f}x+{intercept_x:.2f}\n"
            f"除外: {len(outliers_x)}件 ({ratio_x:.1f}%), 範囲: [{lower_x:.2f}, {upper_x:.2f}]",
            fontsize=10
        )

        # 4つ目：両方除外
        sns.scatterplot(x=subset_xy_filtered[col], y=subset_xy_filtered[target_col], color='orange', alpha=0.5, ax=axes[3])
        sns.lineplot(x=subset_xy_filtered[col], y=y_pred_xy, color='blue', ax=axes[3])
        axes[3].set_title(
            f"X・Y両方3σ除外\n相関: {corr_xy:.2f}, R²: {r2_xy:.2f}\n回帰: y={coef_xy:.2f}x+{intercept_xy:.2f}\n"
            f"除外: Y={len(outliers_y)}件, X={len(outliers_x)}件",
            fontsize=10
        )

        for ax in axes:
            ax.set_xlabel(col)
            ax.set_ylabel(target_col)

        plt.tight_layout()
        filename = os.path.join(save_dir, f"{col}_scatter_4views.png")
        plt.savefig(filename)
        plt.close()
        print(f"Saved: {filename}")


if __name__ == "__main__":
    df =pd.read_csv(r"P:\data\sample.csv",encoding="shift-jis")
    cluster_df = save_top_scatter_plots_seaborn(
        df,
        target_col="Sensor5",
        exclude_cols="Sensor1,Sensor2,Sensor3,Sensor4",
        save_dir=r"P:\data\png",
        top=10,
    )

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os
from sklearn.cluster import AgglomerativeClustering
from sklearn.preprocessing import StandardScaler
from statsmodels.stats.outliers_influence import variance_inflation_factor
plt.rcParams['font.family'] = 'Meiryo'
sns.set(font='Meiryo', style='whitegrid')

def analyze_feature_redundancy(df, target_col, exclude_cols, top=20, n_clusters=5, save_path="feature_redundancy.png"):
    """
    df: pandas DataFrame
    target_col: 目的変数のカラム名（str）
    exclude_cols: 除外対象のカラム名（カンマ区切りのstr）
    top: 相関上位の特徴量数（int）
    n_clusters: クラスタ数（int）
    save_path: 可視化の保存先（str）
    """
    exclude_list = [col.strip() for col in exclude_cols.split(',')]
    feature_cols = [col for col in df.columns if col != target_col and col not in exclude_list]

    # 相関係数で上位特徴量を抽出
    correlations = df[feature_cols].corrwith(df[target_col]).sort_values(key=abs, ascending=False)
    top_features = correlations.head(top).index.tolist()

    # 欠損と定数列を除去
    subset = df[top_features].dropna()
    subset = subset.loc[:, subset.std() != 0]

    # 相関行列 → 距離行列
    corr_matrix = subset.corr().abs()
    distance_matrix = 1 - corr_matrix

    # クラスタリング
    clustering = AgglomerativeClustering(n_clusters=n_clusters, metric='precomputed', linkage='average')
    labels = clustering.fit_predict(distance_matrix)

    # VIF計算
    scaled = StandardScaler().fit_transform(subset)
    vif_values = [variance_inflation_factor(scaled, i) for i in range(scaled.shape[1])]

    # 結果まとめ
    result_df = pd.DataFrame({
        'feature': subset.columns,
        'correlation': correlations[subset.columns].values,
        'cluster': labels,
        'VIF': vif_values
    }).sort_values(by='cluster')

    # 可視化
    plt.figure(figsize=(10, 6))
    sns.scatterplot(data=result_df, x='feature', y='correlation', hue='cluster', size='VIF',
                    palette='tab10', sizes=(50, 300), legend='brief')
    plt.xticks(rotation=45, ha='right')
    plt.title(f"{target_col} に対する特徴量の冗長性分析", font='Meiryo')
    plt.tight_layout()
    os.makedirs(os.path.dirname(save_path), exist_ok=True)
    plt.savefig(save_path)
    plt.close()
    print(f"Saved: {save_path}")

    return result_df

if __name__ == "__main__":
    df = pd.read_csv(r"P:\data\sample.csv", encoding="shift-jis")
    redundancy_df = analyze_feature_redundancy(
        df,
        target_col="Sensor1",
        exclude_cols="Sensor1,Sensor2,Sensor3,Sensor4",
        top=20,
        n_clusters=5,
        save_path=r"P:\data\feature_redundancy.png"
    )

    # クラスタごとの代表特徴量を確認
    print(redundancy_df.groupby('cluster').first())

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from lightgbm import LGBMRegressor
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error, r2_score

class GBDTFeatureEvaluator:
    def __init__(self, df: pd.DataFrame, target_col: str, exclude_cols: str = "", test_size: float = 0.2, random_state: int = 42):
        """
        クラスの初期化：データと設定を受け取り、LightGBMモデルを準備します。

        Parameters:
            df (pd.DataFrame): 入力データフレーム
            target_col (str): 予測対象となる目的変数の列名
            exclude_cols (str): モデルに使わない列名（カンマ区切りで指定）
            test_size (float): 検証データの割合（未使用：CVで分割するため）
            random_state (int): 乱数シード（再現性確保のため）
        """
        self.df = df.copy()
        self.target_col = target_col
        self.exclude_cols = [col.strip() for col in exclude_cols.split(",") if col.strip()]
        self.test_size = test_size
        self.random_state = random_state
        self.model = LGBMRegressor(random_state=self.random_state)
        self.boosters = []  # 各foldのboosterを保存（Gain取得用）

    def prepare_data(self):
        """
        特徴量と目的変数に分割し、除外列を取り除きます。
        """
        drop_cols = self.exclude_cols + [self.target_col]
        self.X = self.df.drop(columns=drop_cols, errors="ignore")
        self.y = self.df[self.target_col]

    def train_model(self, n_splits: int = 5):
        """
        K分割交差検証でLightGBMモデルを学習し、平均スコアと特徴量重要度（Gain/Split）を算出します。

        Parameters:
            n_splits (int): クロスバリデーションの分割数（fold数）
        """
        kf = KFold(n_splits=n_splits, shuffle=True, random_state=self.random_state)
        r2_scores = []
        mse_scores = []
        self.feature_importances_split = np.zeros(self.X.shape[1])  # 使用頻度（Split）
        self.feature_importances_gain = np.zeros(self.X.shape[1])   # 分岐貢献度（Gain）

        for fold, (train_idx, val_idx) in enumerate(kf.split(self.X), 1):
            # 学習・検証データに分割
            X_train, X_val = self.X.iloc[train_idx], self.X.iloc[val_idx]
            y_train, y_val = self.y.iloc[train_idx], self.y.iloc[val_idx]

            # 欠損値処理：前方補完 → 補完できなかった行を削除
            X_train = X_train.fillna(method='ffill').dropna()
            y_train = y_train.loc[X_train.index]
            X_val = X_val.fillna(method='ffill').dropna()
            y_val = y_val.loc[X_val.index]

            # モデル学習
            model = LGBMRegressor(random_state=self.random_state)
            model.fit(X_train, y_train)

            # 検証データで予測・評価
            y_pred = model.predict(X_val)
            r2_scores.append(r2_score(y_val, y_pred))
            mse_scores.append(mean_squared_error(y_val, y_pred))

            # Boosterオブジェクトを保存（Gain取得用）
            booster = model.booster_
            self.boosters.append(booster)

            # 特徴量重要度を加算（後で平均化）
            self.feature_importances_split += booster.feature_importance(importance_type='split')
            self.feature_importances_gain += booster.feature_importance(importance_type='gain')

            print(f"[Fold {fold}] R²: {r2_scores[-1]:.4f}, MSE: {mse_scores[-1]:.4f}")

        # 平均スコアと重要度を出力
        print(f"\n平均 R²: {np.mean(r2_scores):.4f}")
        print(f"平均 MSE: {np.mean(mse_scores):.4f}")
        self.feature_importances_split /= n_splits
        self.feature_importances_gain /= n_splits

    def plot_feature_importance(self, top_n: int = 20):
        features = self.boosters[0].feature_name()
        indices = np.argsort(self.feature_importances_gain)[::-1][:top_n]

        gain_values = self.feature_importances_gain[indices]
        split_values = self.feature_importances_split[indices]
        feature_names = [features[i] for i in indices]

        # コンソール出力
        print(f"\n重要度上位 {top_n} 特徴量一覧（Gainベース）:")
        for i, idx in enumerate(indices):
            print(f"{i+1:2d}位: {features[idx]:<20} → Gain: {self.feature_importances_gain[idx]:.2f}, Split: {self.feature_importances_split[idx]:.0f}")

        # グラフ描画（棒グラフ＋折れ線グラフ）
        fig, ax1 = plt.subplots(figsize=(12, 6))

        ax1.bar(feature_names, gain_values, color='steelblue', label='Gain')
        ax1.set_ylabel('重要度（Gain）', fontname='MS Gothic', color='steelblue')
        ax1.tick_params(axis='y', labelcolor='steelblue')
        ax1.set_xticklabels(feature_names, rotation=90, fontname='MS Gothic')
        ax1.set_title('特徴量の重要度と使用頻度（Gain + Split）', fontname='MS Gothic')

        # 第2軸（Split）
        ax2 = ax1.twinx()
        ax2.plot(feature_names, split_values, color='darkorange', marker='o', label='Split')
        ax2.set_ylabel('使用頻度（Split）', fontname='MS Gothic', color='darkorange')
        ax2.tick_params(axis='y', labelcolor='darkorange')

        fig.tight_layout()
        plt.show()

    def run(self, n_splits: int = 5, top_n: int = 20):
        """
        一連の処理を実行：データ準備 → モデル学習 → 重要度可視化

        Parameters:
            n_splits (int): クロスバリデーションの分割数
            top_n (int): 表示する特徴量の上位件数
        """
        self.prepare_data()
        self.train_model(n_splits=n_splits)
        self.plot_feature_importance(top_n=top_n)

if __name__ == "__main__":
    df = pd.read_csv(r"P:\data\sample.csv", encoding="shift-jis")
    evaluator = GBDTFeatureEvaluator(
        df=df,
        target_col="Sensor5",
        exclude_cols="Sensor1,Sensor2,Sensor3,Sensor4"
    )
    evaluator.run(n_splits=5, top_n=20)