【Python実践】アンサンブル回帰で効く特徴量を自動選定！Forward/Backwardの実践ガイド（コピペで使えるサンプルコード付き）

from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
import numpy as np

def forward_selection(X, y, model, scoring='neg_root_mean_squared_error', max_features=None, cv=5, verbose=True):
    """
    Forward Selection による特徴量選定

    Parameters:
    ----------
    X : pd.DataFrame
        特徴量データ
    y : pd.Series or np.array
        目的変数
    model : sklearn互換の回帰モデル
        例：LGBMRegressor(), XGBRegressor(), etc.
    scoring : str
        スコアリング指標（sklearnのcross_val_scoreと同じ形式）
    max_features : int or None
        選定する最大特徴量数（Noneなら全特徴量が対象）
    cv : int
        クロスバリデーションの分割数
    verbose : bool
        Trueなら進捗を表示

    Returns:
    -------
    selected_features : list
        選ばれた特徴量のリスト
    scores : list
        各ステップでのスコア
    """
    remaining = list(X.columns)
    selected = []
    best_score = -np.inf
    scores = []

    while remaining:
        scores_candidates = []
        for feature in remaining:
            trial_features = selected + [feature]
            score = cross_val_score(model, X[trial_features], y, scoring=scoring, cv=cv).mean()
            scores_candidates.append((score, feature))

        scores_candidates.sort(reverse=True)
        best_candidate_score, best_candidate = scores_candidates[0]

        if verbose:
            print(f"Trying: {best_candidate}, Score: {best_candidate_score:.5f}")

        if best_candidate_score > best_score:
            selected.append(best_candidate)
            remaining.remove(best_candidate)
            best_score = best_candidate_score
            scores.append(best_candidate_score)
        else:
            break

        if max_features and len(selected) >= max_features:
            break

    return selected, scores


if __name__ == "__main__":

    # =======================================================
    # ダミーデータの作成
    # =======================================================
    import pandas as pd
    from sklearn.datasets import make_regression

    # 1000サンプル、10特徴量のダミーデータを作成
    X_array, y = make_regression(n_samples=1000, n_features=10, noise=0.1, random_state=42)
    X = pd.DataFrame(X_array, columns=[f"feature_{i}" for i in range(X_array.shape[1])])

    # =======================================================
    # Forward Selectionの実行
    # =======================================================
    from lightgbm import LGBMRegressor

    # 例：モデルとデータを用意（X, y は事前に定義されている前提）
    model = LGBMRegressor()
    selected_features, scores = forward_selection(X, y, model)

    print("選ばれた特徴量:", selected_features)
    print("各ステップのスコア:", scores)

from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
import numpy as np

def backward_elimination(X, y, model, scoring='neg_root_mean_squared_error', min_features=1, cv=5, verbose=True):
    """
    Backward Elimination による特徴量選定

    Parameters:
    ----------
    X : pd.DataFrame
        特徴量データ
    y : pd.Series or np.array
        目的変数
    model : sklearn互換の回帰モデル
    scoring : str
        スコアリング指標（sklearn形式）
    min_features : int
        最低限残す特徴量数
    cv : int
        クロスバリデーションの分割数
    verbose : bool
        Trueなら進捗を表示

    Returns:
    -------
    selected_features : list
        最終的に残った特徴量
    scores : list
        各ステップでのスコア
    """
    selected = list(X.columns)
    scores = []
    best_score = cross_val_score(model, X[selected], y, scoring=scoring, cv=cv).mean()
    scores.append(best_score)

    while len(selected) > min_features:
        scores_candidates = []
        for feature in selected:
            trial_features = [f for f in selected if f != feature]
            score = cross_val_score(model, X[trial_features], y, scoring=scoring, cv=cv).mean()
            scores_candidates.append((score, feature))

        scores_candidates.sort(reverse=True)
        best_candidate_score, worst_feature = scores_candidates[0]

        if verbose:
            print(f"Removing: {worst_feature}, Score: {best_candidate_score:.5f}")

        if best_candidate_score >= best_score:
            selected.remove(worst_feature)
            best_score = best_candidate_score
            scores.append(best_candidate_score)
        else:
            break

    return selected, scores


if __name__ == "__main__":

    # =======================================================
    # ダミーデータの作成
    # =======================================================
    import pandas as pd
    from sklearn.datasets import make_regression

    # 1000サンプル、10特徴量のダミーデータを作成
    X_array, y = make_regression(n_samples=1000, n_features=10, noise=0.1, random_state=42)
    X = pd.DataFrame(X_array, columns=[f"feature_{i}" for i in range(X_array.shape[1])])

    # =======================================================
    # Backward Eliminationの実行
    # =======================================================
    from lightgbm import LGBMRegressor
    import pandas as pd


    # モデル定義
    model = LGBMRegressor()

    # 特徴量選定
    selected_features, scores = backward_elimination(X, y, model)

    print("残った特徴量:", selected_features)
    print("各ステップのスコア:", scores)