gp_sampler_example.main()

Complexity

Conditions

1

Size

Total Lines	107
Code Lines	26

Duplication

Lines	0
Ratio	0 %

Importance

Changes

0

"""
def main():
    # === GPSampler Example ===
    # Gaussian Process Bayesian Optimization

    gaussian_process_theory()

    # Load dataset - classification problem
    X, y = load_breast_cancer(return_X_y=True)
    print(
        f"Dataset: Breast cancer classification ({X.shape[0]} samples, {X.shape[1]} features)"
    )

    # Create experiment
    estimator = SVC(random_state=42)
    experiment = SklearnCvExperiment(estimator=estimator, X=X, y=y, cv=5)

    # Define search space - mixed parameter types
    param_space = {
        "C": (0.01, 100),  # Continuous - regularization
        "gamma": (1e-6, 1e2),  # Continuous - RBF parameter
        "kernel": ["rbf", "poly", "sigmoid"],  # Categorical
        "degree": (2, 5),  # Integer - polynomial degree
        "coef0": (0.0, 1.0),  # Continuous - kernel coefficient
    }

    # Search Space (Mixed parameter types):
    # for param, space in param_space.items():
    #   print(f"  {param}: {space}")

    # Configure GPSampler
    optimizer = GPSampler(
        param_space=param_space,
        n_trials=25,  # Fewer trials - GP is sample efficient
        random_state=42,
        experiment=experiment,
        n_startup_trials=8,  # Random initialization before GP modeling
        deterministic_objective=False,  # Set True if objective is noise-free
    )

    # GPSampler Configuration:
    # n_trials: configured above
    # n_startup_trials: random initialization
    # deterministic_objective: configures noise handling
    # Acquisition function: Expected Improvement (default)

    # Run optimization
    # Running GP-based optimization...
    best_params = optimizer.run()

    # Results
    print("\n=== Results ===")
    print(f"Best parameters: {best_params}")
    print(f"Best score: {optimizer.best_score_:.4f}")
    print()

    # GP Optimization Phases:
    #
    # Phase 1 (Trials 1-8): Random Exploration
    #  Random sampling for initial GP training data
    #  Builds diverse set of observations
    #  No model assumptions yet

    # Phase 2 (Trials 9-25): GP-guided Search
    #  GP model learns from observed data
    #  Acquisition function balances:
    #   - Exploitation: areas with high predicted performance
    #   - Exploration: areas with high uncertainty
    #  Sequential decision making with uncertainty

    # GP Model Characteristics:
    #  Handles mixed parameter types (continuous, discrete, categorical)
    #  Provides uncertainty estimates for all predictions
    #  Automatically balances exploration vs exploitation
    #  Sample efficient - good for expensive evaluations
    #  Can incorporate prior knowledge through mean/kernel functions

    # Acquisition Function Behavior:
    #  High mean + low variance → exploitation
    #  Low mean + high variance → exploration
    #  Balanced trade-off prevents premature convergence
    #  Adapts exploration strategy based on observed data

    # Best Use Cases:
    #  Expensive objective function evaluations
    #  Small to medium parameter spaces (< 20 dimensions)
    #  When uncertainty quantification is valuable
    #  Mixed parameter types (continuous + categorical)
    #  Noisy objective functions (with appropriate kernel)

    # Limitations:
    #  Computational cost grows with number of observations
    #  Hyperparameter tuning for GP kernel
    #  May struggle in very high dimensions
    #  Assumes some smoothness in objective function

    # Comparison with TPESampler:
    # GPSampler advantages:
    #   + Principled uncertainty quantification
    #   + Better for expensive evaluations
    #   + Can handle constraints naturally
    #
    # TPESampler advantages:
    #   + Faster computation
    #   + Better scalability to high dimensions
    #   + More robust hyperparameter defaults

    return best_params, optimizer.best_score_