import pandas as pd
import numpy as np

# Load
train = pd.read_parquet("/content/Train.parquet")
test  = pd.read_parquet("/content/Test.parquet")

# Safe datetime parsing
for c in ["DATE", "TRIAL_CONVERTED_TO_PURCHASE_DATE"]:
    if c in train.columns:
        train[c] = pd.to_datetime(train[c], errors="coerce")
    if c in test.columns:
        test[c] = pd.to_datetime(test[c], errors="coerce")

print("Loaded successfully.")
print("Train shape:", train.shape, "| Test shape:", test.shape)

# Tiny at-a-glance summary
def tiny_overview(df):
    out = {
        "rows": len(df),
        "cols": df.shape[1],
        "date_min": df["DATE"].min() if "DATE" in df.columns else None,
        "date_max": df["DATE"].max() if "DATE" in df.columns else None,
    }
    return pd.Series(out)

display(pd.DataFrame({"train": tiny_overview(train), "test": tiny_overview(test)}))

# Optional quick peek (2 rows only)
display(train.head(2))
display(test.head(2))

Loaded successfully.
Train shape: (66452, 35) | Test shape: (6898, 35)

# 120-day label + time sanity

import numpy as np
import pandas as pd

# datetime coercion (idempotent)
for c in ["DATE", "TRIAL_CONVERTED_TO_PURCHASE_DATE"]:
    if c in train.columns:
        train[c] = pd.to_datetime(train[c], errors="coerce")
    if c in test.columns:
        test[c] = pd.to_datetime(test[c], errors="coerce")

# normalize raw label to 0/1 if needed
def to_bool_int(s):
    if s.dtype == "bool":
        return s.astype(int)
    if s.dtype == "object":
        return s.astype(str).str.lower().map({"true":1, "false":0}).fillna(0).astype(int)
    return s.astype(int)

for df in (train, test):
    if "IS_TRIAL_CONVERTED_TO_PURCHASE" in df.columns:
        df["IS_TRIAL_CONVERTED_TO_PURCHASE"] = to_bool_int(df["IS_TRIAL_CONVERTED_TO_PURCHASE"])

# days_to_convert (train + test)
for df in (train, test):
    if {"DATE","TRIAL_CONVERTED_TO_PURCHASE_DATE"}.issubset(df.columns):
        df["days_to_convert"] = (df["TRIAL_CONVERTED_TO_PURCHASE_DATE"] - df["DATE"]).dt.days
        df.loc[df["days_to_convert"] < 0, "days_to_convert"] = np.nan
    else:
        df["days_to_convert"] = np.nan

# precise 120-day target (train + test, but I won't use test label for training)
for df in (train, test):
    if "IS_TRIAL_CONVERTED_TO_PURCHASE" in df.columns:
        df["LABEL_120D"] = np.where(
            (df["IS_TRIAL_CONVERTED_TO_PURCHASE"] == 1) & (df["days_to_convert"].le(120)),
            1, 0
        ).astype(int)
    else:
        df["LABEL_120D"] = np.nan

# month key for later aggregation
for df in (train, test):
    if "DATE" in df.columns:
        df["trial_month"] = df["DATE"].dt.to_period("M").astype(str)

# stash test label for final evaluation only
y_test_holdout = test["LABEL_120D"].copy()

# sanity print
print({
    "train_raw_pos": int(train["IS_TRIAL_CONVERTED_TO_PURCHASE"].sum()),
    "train_pos_120d": int(train["LABEL_120D"].sum()),
    "test_raw_pos": int(test["IS_TRIAL_CONVERTED_TO_PURCHASE"].sum()),
    "test_pos_120d": int(test["LABEL_120D"].sum())
})

{'train_raw_pos': 3903, 'train_pos_120d': 3749, 'test_raw_pos': 501, 'test_pos_120d': 484}

train.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 66452 entries, 0 to 66451
Data columns (total 38 columns):
 #   Column                            Non-Null Count  Dtype         
---  ------                            --------------  -----         
 0   CHANNEL_ID                        66452 non-null  int64         
 1   CAMPAIGN_ID                       66452 non-null  int64         
 2   DATE                              66452 non-null  datetime64[ns]
 3   TRIAL_CONVERTED_TO_PURCHASE_DATE  3903 non-null   datetime64[ns]
 4   IS_TRIAL_CONVERTED_TO_PURCHASE    66452 non-null  int64         
 5   NUM_OF_INSTALLED_AGENTS           66452 non-null  float64       
 6   IP_COUNTRY                        66452 non-null  object        
 7   REGISTRATION_EMAIL_TYPE           66452 non-null  object        
 8   COMPANY_TYPE                      57830 non-null  object        
 9   INITIAL_ATERA_GOAL                9809 non-null   object        
 10  TECH_EMPLOYEES_RANGE              57850 non-null  object        
 11  PREVIOUS_RMM_EXPERIENCE           9799 non-null   object        
 12  IS_SURVEY_COMPLETED               66452 non-null  bool          
 13  BUSINESS_REGION                   65912 non-null  object        
 14  'customer created'                1520 non-null   float64       
 15  'ticket generated'                2030 non-null   float64       
 16  'agent downloaded'                27194 non-null  float64       
 17  'report generated'                4130 non-null   float64       
 18  'remote connection- success'      8058 non-null   float64       
 19  'threshold Profile created'       0 non-null      float64       
 20  'it automation profile created'   5446 non-null   float64       
 21  'run script'                      1061 non-null   float64       
 22  'alerts email set'                189 non-null    float64       
 23  'invoice generate'                77 non-null     float64       
 24  'discovered network'              0 non-null      float64       
 25  'technician added'                2049 non-null   float64       
 26  CLEARBIT_SECTOR                   23791 non-null  object        
 27  CLEARBIT_INDUSTRY_GROUP           23794 non-null  object        
 28  CLEARBIT_INDUSTRY                 23791 non-null  object        
 29  VIEWED_PRICING_PAGE               66452 non-null  bool          
 30  HUBSPOT_CLICKED                   63786 non-null  float64       
 31  SEGMENT_EVENTS                    63927 non-null  float64       
 32  DISTINCT_SEGMENT_EVENTS           63927 non-null  float64       
 33  SEGMENT_PAGE_VIEWS                52931 non-null  float64       
 34  DISTINCT_SEGMENT_PAGE_VIEWS       52931 non-null  float64       
 35  days_to_convert                   3903 non-null   float64       
 36  LABEL_120D                        66452 non-null  int64         
 37  trial_month                       66452 non-null  object        
dtypes: bool(2), datetime64[ns](2), float64(19), int64(4), object(11)
memory usage: 18.4+ MB

# Canonicalize column names (once for both train & test)
def clean_col(c: str) -> str:
    c = str(c).strip()
    if c.startswith("'") and c.endswith("'"):
        c = c[1:-1]                # drop surrounding single quotes
    c = c.replace(" ", "_").replace("-", "_")
    return c.lower()

train.rename(columns={c: clean_col(c) for c in train.columns}, inplace=True)
test.rename(columns={c: clean_col(c) for c in test.columns}, inplace=True)

for df in (train, test):
    if "label_120d" not in df.columns and "LABEL_120D" in df.columns:
        df.rename(columns={"LABEL_120D": "label_120d"}, inplace=True)
    if "days_to_convert" not in df.columns and "days_to_convert" in df.columns:
        pass
    if "trial_month" not in df.columns and "trial_month" in df.columns:
        pass

# Rebuild feature groups with cleaned names

dates_features = ["date", "trial_converted_to_purchase_date"]
target_col     = "label_120d"         # THE target (not a feature)
raw_label_col  = "is_trial_converted_to_purchase"  # keep tracked but ignored for X

business_features = [
    "company_type", "initial_atera_goal", "previous_rmm_experience",
    "clearbit_sector", "clearbit_industry_group", "clearbit_industry",
    "tech_employees_range", "business_region", "ip_country",
    "registration_email_type", "num_of_installed_agents", "is_survey_completed"
]

marketing_features = ["channel_id", "campaign_id", "viewed_pricing_page", "hubspot_clicked"]

engagement_features = [
    "segment_events", "distinct_segment_events",
    "segment_page_views", "distinct_segment_page_views"
]

behavioral_features = [
    "customer_created", "ticket_generated", "agent_downloaded", "report_generated",
    "remote_connection__success",
    "it_automation_profile_created", "run_script",
    "alerts_email_set", "invoice_generate", "technician_added",
    "threshold_profile_created", "discovered_network"
]

# helpers not to be used as features (except trial_month for aggregation later)
engineered_helpers = ["days_to_convert", "trial_month"]

# Coverage check
grouped = (
    dates_features + [target_col, raw_label_col] +
    business_features + marketing_features +
    engagement_features + behavioral_features +
    engineered_helpers
)

print("Total columns in df:", train.shape[1])
print("Total columns mapped:", len(grouped))
print("Missing in mapping:", [c for c in train.columns if c not in grouped])
print("Duplicates in mapping:", [c for c in grouped if grouped.count(c) > 1])

Total columns in df: 38
Total columns mapped: 38
Missing in mapping: []
Duplicates in mapping: []

from IPython.display import display, HTML

# helper: render tags only for columns that actually exist
def tag_spans(cols, available):
    cols = [c for c in cols if c in available]
    return "\n".join([f'<span class="tag">{c}</span>' for c in cols])

available = set(train.columns)  # assumes you've already cleaned names (lowercase, underscores)

# Groups (cleaned names)
dates_features = ["date", "trial_converted_to_purchase_date"]
target_feature = ["is_trial_converted_to_purchase"]  # raw label (kept for reference, not used as X)

behavioral_features = [
    "customer_created", "ticket_generated", "agent_downloaded",
    "report_generated", "remote_connection__success",
    "it_automation_profile_created", "run_script",
    "alerts_email_set", "invoice_generate", "technician_added",
    "threshold_profile_created", "discovered_network"
]

business_features = [
    "company_type", "initial_atera_goal", "previous_rmm_experience",
    "clearbit_sector", "clearbit_industry_group", "clearbit_industry",
    "tech_employees_range", "business_region", "ip_country",
    "registration_email_type", "num_of_installed_agents", "is_survey_completed"
]

marketing_features = ["channel_id", "campaign_id", "viewed_pricing_page", "hubspot_clicked"]

engagement_features = [
    "segment_events", "distinct_segment_events",
    "segment_page_views", "distinct_segment_page_views"
]

# engineered / target helpers (not used as X, except trial_month for aggregation later)
engineered_helpers = ["days_to_convert", "label_120d", "trial_month"]

html_table = f"""
<style>
.feature-box {{
    border: 1px solid #ddd; border-radius: 12px; padding: 18px; margin: 12px 0;
    background-color: #fafafa; font-family: 'Arial', sans-serif;
}}
.feature-title {{ font-weight: bold; font-size: 16px; color: #1A5276; }}
.feature-desc {{ margin-top: 8px; color: #333; font-size: 14px; }}
.tag {{
    background-color: #f0f0f0; border-radius: 8px; padding: 4px 8px; margin: 3px;
    display: inline-block; font-size: 13px; color: #2c3e50; font-family: monospace;
}}
</style>

<div class="feature-box">
  <div class="feature-title">🟦 User Behavioral Features</div>
  <div class="feature-desc">
    Actions performed during the trial that reflect engagement. Missing values typically mean the action did not occur.
  </div>
  <div>
    {tag_spans(behavioral_features, available)}
  </div>
</div>

<div class="feature-box">
  <div class="feature-title">🟧 Business / Contextual Features</div>
  <div class="feature-desc">
    Company-level attributes and Clearbit enrichment (sector, industry, team size, region).
  </div>
  <div>
    {tag_spans(business_features, available)}
  </div>
</div>

<div class="feature-box">
  <div class="feature-title">🟩 Marketing / Campaign Features</div>
  <div class="feature-desc">
    Channel and campaign identifiers plus pricing/HubSpot interactions.
  </div>
  <div>
    {tag_spans(marketing_features, available)}
  </div>
</div>

<div class="feature-box">
  <div class="feature-title">🟨 Engagement Metrics</div>
  <div class="feature-desc">
    Quantitative activity signals such as tracked events and page views.
  </div>
  <div>
    {tag_spans(engagement_features, available)}
  </div>
</div>

<div class="feature-box">
  <div class="feature-title">⬜ Dates & Raw Target</div>
  <div class="feature-desc">
    Registration and conversion dates, and the original label kept for reference (not used as features).
  </div>
  <div>
    {tag_spans(dates_features + target_feature, available)}
  </div>
</div>

<div class="feature-box">
  <div class="feature-title">🟪 Engineered / Target Helpers</div>
  <div class="feature-desc">
    Helper fields used for the 120-day target and monthly aggregation. <b>Not used as model inputs.</b>
  </div>
  <div>
    {tag_spans(engineered_helpers, available)}
  </div>
</div>
"""

display(HTML(html_table))

# Missing values audit

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# safety: keep dates as datetime (idempotent)
for c in ["date", "trial_converted_to_purchase_date"]:
    if c in train.columns:
        train[c] = pd.to_datetime(train[c], errors="coerce")

# never touch these
protect = {"label_120d", "is_trial_converted_to_purchase", "days_to_convert", "trial_month"}

# drop columns that are 100% missing (except protected)
all_null = [c for c in train.columns if c not in protect and train[c].isna().all()]
if all_null:
    print("Dropped 100% missing:", all_null)
    train.drop(columns=all_null, inplace=True)
else:
    print("No 100% missing columns.")

print("Updated shape:", train.shape)

# build missing summary excluding targets/helpers
exclude = protect
cols_for_missing = [c for c in train.columns if c not in exclude]
missing_percent = (train[cols_for_missing].isna().sum() / len(train)) * 100
missing_percent = missing_percent[missing_percent > 0].sort_values(ascending=False)

print("\nTop 15 missing (%):\n", missing_percent.head(15).round(2))

# feature groups (clean names)
behavioral_features = [
    "customer_created","ticket_generated","agent_downloaded","report_generated",
    "remote_connection__success","it_automation_profile_created","run_script",
    "alerts_email_set","invoice_generate","technician_added",
    "threshold_profile_created","discovered_network"  # may already be dropped
]
business_features = [
    "company_type","initial_atera_goal","previous_rmm_experience",
    "clearbit_sector","clearbit_industry_group","clearbit_industry",
    "tech_employees_range","business_region","ip_country",
    "registration_email_type","num_of_installed_agents","is_survey_completed"
]
date_features = ["date","trial_converted_to_purchase_date"]

# color by group
colors = []
for col in missing_percent.head(15).index:
    if col in behavioral_features: colors.append("#5DADE2")      # behavioral
    elif col in business_features: colors.append("#F5B041")       # business
    elif col in date_features:     colors.append("#BFC9CA")       # dates
    else:                          colors.append("#D5DBDB")       # other

# plot
plt.figure(figsize=(10, 5))
ax = missing_percent.head(15).plot(kind="bar", color=colors, edgecolor="gray")
plt.title("Top 15 Columns with Missing Values (%)")
plt.ylabel("Percent of Missing Values")
plt.xticks(rotation=45, ha="right")
for i, v in enumerate(missing_percent.head(15).round(1)):
    ax.text(i, v + 1, f"{v:.1f}%", ha="center", fontsize=9)

plt.figlegend(
    handles=[
        plt.Line2D([0],[0], color="#5DADE2", lw=6, label="User Behavioral"),
        plt.Line2D([0],[0], color="#F5B041", lw=6, label="Business / Context"),
        plt.Line2D([0],[0], color="#BFC9CA", lw=6, label="Dates"),
    ],
    loc="lower center", ncol=3, bbox_to_anchor=(0.5,-0.25)
)
plt.tight_layout()
plt.show()

Dropped 100% missing: ['threshold_profile_created', 'discovered_network']
Updated shape: (66452, 36)

Top 15 missing (%):
 invoice_generate                    99.88
alerts_email_set                    99.72
run_script                          98.40
customer_created                    97.71
ticket_generated                    96.95
technician_added                    96.92
trial_converted_to_purchase_date    94.13
report_generated                    93.78
it_automation_profile_created       91.80
remote_connection__success          87.87
previous_rmm_experience             85.25
initial_atera_goal                  85.24
clearbit_industry                   64.20
clearbit_sector                     64.20
clearbit_industry_group             64.19
dtype: float64

# anchors for the heuristic
train["has_business_email"] = train["registration_email_type"].astype(str).str.lower().str.contains(
    r"business|company|work|corp|b2b", regex=True, na=False
).astype(int)
test["has_business_email"] = test["registration_email_type"].astype(str).str.lower().str.contains(
    r"business|company|work|corp|b2b", regex=True, na=False
).astype(int)

train["agent_presence"] = (train["num_of_installed_agents"].fillna(0) >= 1).astype(int)
test["agent_presence"]  = (test["num_of_installed_agents"].fillna(0)  >= 1).astype(int)

train["agents_bin"] = pd.cut(train["num_of_installed_agents"].fillna(0), [-1,0,1,4,9999], labels=["0","1","2-4","5+"])
test["agents_bin"]  = pd.cut(test["num_of_installed_agents"].fillna(0),  [-1,0,1,4,9999], labels=["0","1","2-4","5+"])

# compact behavioral intensity
behav_cols = [
    "customer_created","ticket_generated","agent_downloaded","report_generated",
    "remote_connection__success","it_automation_profile_created","run_script",
    "alerts_email_set","invoice_generate","technician_added"
]
for df in (train, test):
    for c in behav_cols:
        if c in df.columns:
            df[c] = (df[c].fillna(0) > 0).astype(int)
    df["event_count"] = df[ [c for c in behav_cols if c in df.columns] ].sum(axis=1)

# simple intent ratios
for df in (train, test):
    df["events_per_view"] = df["event_count"] / (1 + df.get("segment_page_views", pd.Series(0, index=df.index)).fillna(0))
    df["pricing_intent"]  = df["viewed_pricing_page"].astype(int) if "viewed_pricing_page" in df.columns else 0
    df["email_x_agents"]  = df["has_business_email"] * df["agent_presence"]

# Feature lists & forbidden

# numeric (continuous counts/metrics)
num_cols = [c for c in [
    "num_of_installed_agents", "segment_events", "distinct_segment_events",
    "segment_page_views", "distinct_segment_page_views", "hubspot_clicked",
    # engineered numeric
    "event_count", "events_per_view"
] if c in train.columns]

# binary flags / behavioral events (0/1 semantics)
bin_cols = [c for c in [
    # engineered anchors + intent + interaction
    "has_business_email", "agent_presence", "pricing_intent", "email_x_agents",
    # original behavior flags (coerced to 0/1)
    "is_survey_completed", "customer_created", "ticket_generated", "agent_downloaded",
    "report_generated", "remote_connection__success", "it_automation_profile_created",
    "run_script", "alerts_email_set", "invoice_generate", "technician_added"
] if c in train.columns]

# categorical (to one-hot) — include engineered 'agents_bin'
cat_cols = [c for c in [
    "agents_bin",
    "company_type", "initial_atera_goal", "previous_rmm_experience",
    "clearbit_sector", "clearbit_industry_group", "clearbit_industry",
    "tech_employees_range", "business_region", "ip_country",
    "registration_email_type", "channel_id", "campaign_id"
] if c in train.columns]

# columns that must NOT enter X (targets, helpers, time-leaky fields)
ignore_cols = {
    "label_120d", "is_trial_converted_to_purchase", "days_to_convert",
    "trial_converted_to_purchase_date", "trial_month"
}

# canonical feature list
feature_cols = [c for c in train.columns if c not in ignore_cols and c in (num_cols + bin_cols + cat_cols)]
print("num/bin/cat:", len(num_cols), len(bin_cols), len(cat_cols))

num/bin/cat: 8 15 13

# Time-based split (last 3 months as validation)
months_sorted = train["trial_month"].sort_values().unique()
cut_month = months_sorted[-3]
is_val = train["trial_month"] >= cut_month

X_tr  = train.loc[~is_val, feature_cols].copy()
y_tr  = train.loc[~is_val, "label_120d"].astype(int)
X_val = train.loc[ is_val, feature_cols].copy()
y_val = train.loc[ is_val, "label_120d"].astype(int)

print({"train_rows": len(X_tr), "val_rows": len(X_val), "val_months": sorted(train.loc[is_val, "trial_month"].unique())})

{'train_rows': 59318, 'val_rows': 7134, 'val_months': ['2024-10', '2024-11', '2024-12']}

# Baseline heuristic: "Business email + 1+ installed agents"


for df in (train, test):
    # baseline_pred = 1 only when BOTH conditions are true
    df.loc[:, "baseline_pred"] = (
        (df["has_business_email"] == 1) &
        (df["agent_presence"] == 1)
    ).astype(int)

# Preprocess + Logistic pipeline (baseline)

from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression

base_preprocess = ColumnTransformer(
    transformers=[
        ("num", Pipeline([
            ("imp", SimpleImputer(strategy="median")),
            ("sc",  StandardScaler())
        ]), num_cols),
        ("bin", SimpleImputer(strategy="most_frequent"), bin_cols),
        ("cat", Pipeline([
            ("imp", SimpleImputer(strategy="most_frequent")),
            ("ohe", OneHotEncoder(handle_unknown="ignore", min_frequency=30))
            # consider min_frequency=100 if you see leakage from rare IDs
        ]), cat_cols),
    ],
    remainder="drop",
    sparse_threshold=0.3
)

logreg_pipe = Pipeline([
    ("prep", base_preprocess),
    ("clf", LogisticRegression(max_iter=1000, class_weight="balanced", random_state=42))
])

from sklearn import set_config
set_config(display="diagram")
logreg_pipe

Pipeline(steps=[('prep',
                 ColumnTransformer(transformers=[('num',
                                                  Pipeline(steps=[('imp',
                                                                   SimpleImputer(strategy='median')),
                                                                  ('sc',
                                                                   StandardScaler())]),
                                                  ['num_of_installed_agents',
                                                   'segment_events',
                                                   'distinct_segment_events',
                                                   'segment_page_views',
                                                   'distinct_segment_page_views',
                                                   'hubspot_clicked',
                                                   'event_count',
                                                   'events_per_view']),
                                                 ('bin',
                                                  SimpleImputer(strategy='...
                                                                                 min_frequency=30))]),
                                                  ['agents_bin', 'company_type',
                                                   'initial_atera_goal',
                                                   'previous_rmm_experience',
                                                   'clearbit_sector',
                                                   'clearbit_industry_group',
                                                   'clearbit_industry',
                                                   'tech_employees_range',
                                                   'business_region',
                                                   'ip_country',
                                                   'registration_email_type',
                                                   'channel_id',
                                                   'campaign_id'])])),
                ('clf',
                 LogisticRegression(class_weight='balanced', max_iter=1000,
                                    random_state=42))])

Pipeline(steps=[('prep',
                 ColumnTransformer(transformers=[('num',
                                                  Pipeline(steps=[('imp',
                                                                   SimpleImputer(strategy='median')),
                                                                  ('sc',
                                                                   StandardScaler())]),
                                                  ['num_of_installed_agents',
                                                   'segment_events',
                                                   'distinct_segment_events',
                                                   'segment_page_views',
                                                   'distinct_segment_page_views',
                                                   'hubspot_clicked',
                                                   'event_count',
                                                   'events_per_view']),
                                                 ('bin',
                                                  SimpleImputer(strategy='...
                                                                                 min_frequency=30))]),
                                                  ['agents_bin', 'company_type',
                                                   'initial_atera_goal',
                                                   'previous_rmm_experience',
                                                   'clearbit_sector',
                                                   'clearbit_industry_group',
                                                   'clearbit_industry',
                                                   'tech_employees_range',
                                                   'business_region',
                                                   'ip_country',
                                                   'registration_email_type',
                                                   'channel_id',
                                                   'campaign_id'])])),
                ('clf',
                 LogisticRegression(class_weight='balanced', max_iter=1000,
                                    random_state=42))])

ColumnTransformer(transformers=[('num',
                                 Pipeline(steps=[('imp',
                                                  SimpleImputer(strategy='median')),
                                                 ('sc', StandardScaler())]),
                                 ['num_of_installed_agents', 'segment_events',
                                  'distinct_segment_events',
                                  'segment_page_views',
                                  'distinct_segment_page_views',
                                  'hubspot_clicked', 'event_count',
                                  'events_per_view']),
                                ('bin', SimpleImputer(strategy='most_frequent'),
                                 ['has_bu...
                                                  SimpleImputer(strategy='most_frequent')),
                                                 ('ohe',
                                                  OneHotEncoder(handle_unknown='ignore',
                                                                min_frequency=30))]),
                                 ['agents_bin', 'company_type',
                                  'initial_atera_goal',
                                  'previous_rmm_experience', 'clearbit_sector',
                                  'clearbit_industry_group',
                                  'clearbit_industry', 'tech_employees_range',
                                  'business_region', 'ip_country',
                                  'registration_email_type', 'channel_id',
                                  'campaign_id'])])

['num_of_installed_agents', 'segment_events', 'distinct_segment_events', 'segment_page_views', 'distinct_segment_page_views', 'hubspot_clicked', 'event_count', 'events_per_view']

SimpleImputer(strategy='median')

StandardScaler()

['has_business_email', 'agent_presence', 'pricing_intent', 'email_x_agents', 'is_survey_completed', 'customer_created', 'ticket_generated', 'agent_downloaded', 'report_generated', 'remote_connection__success', 'it_automation_profile_created', 'run_script', 'alerts_email_set', 'invoice_generate', 'technician_added']

SimpleImputer(strategy='most_frequent')

['agents_bin', 'company_type', 'initial_atera_goal', 'previous_rmm_experience', 'clearbit_sector', 'clearbit_industry_group', 'clearbit_industry', 'tech_employees_range', 'business_region', 'ip_country', 'registration_email_type', 'channel_id', 'campaign_id']

SimpleImputer(strategy='most_frequent')

# Sanity checks (no leakage) + smoke test
import numpy as np

forbidden = {"label_120d","is_trial_converted_to_purchase","days_to_convert","trial_converted_to_purchase_date","trial_month"}
assert not any(c in forbidden for c in feature_cols), "Forbidden columns in X!"

missing_train = [c for c in feature_cols if c not in train.columns]
missing_test  = [c for c in feature_cols if c not in test.columns]
print("Missing in TRAIN:", missing_train, " | Missing in TEST:", missing_test)
for c in missing_test:
    test[c] = np.nan

# smoke: ensure transformer runs
_ = base_preprocess.fit_transform(X_tr.iloc[:200])
print("Preprocess.fit_transform[:200] -> OK")

Missing in TRAIN: []  | Missing in TEST: []
Preprocess.fit_transform[:200] -> OK

import matplotlib.pyplot as plt
from sklearn.metrics import (
    precision_recall_curve, roc_curve,
    average_precision_score, roc_auc_score, brier_score_loss
)
from sklearn.calibration import calibration_curve

plt.style.use("seaborn-v0_8-whitegrid")

def setup_figure(title):
    plt.figure(figsize=(7, 6))
    plt.title(title, fontsize=16, weight='bold')
    plt.xlabel("", fontsize=14)
    plt.ylabel("", fontsize=14)

# Fit model
logreg_pipe.fit(X_tr, y_tr)
p_val = logreg_pipe.predict_proba(X_val)[:, 1]

# ---- PR Curve ----
prec, rec, _ = precision_recall_curve(y_val, p_val)
plt.figure(figsize=(7, 6))
plt.plot(rec, prec, linewidth=2.5, color="#1B84E7")
plt.xlabel("Recall", fontsize=14)
plt.ylabel("Precision", fontsize=14)
plt.title("Precision–Recall Curve (Validation)", fontsize=16, weight="bold")
plt.grid(alpha=0.25)
plt.show()

# ---- ROC Curve ----
fpr, tpr, _ = roc_curve(y_val, p_val)
plt.figure(figsize=(7, 6))
plt.plot(fpr, tpr, linewidth=2.5, color="#2ECC71", label="ROC Curve")
plt.plot([0, 1], [0, 1], linestyle="--", color="#7f8c8d", linewidth=1.8)
plt.xlabel("False Positive Rate (FPR)", fontsize=14)
plt.ylabel("True Positive Rate (TPR)", fontsize=14)
plt.title("ROC Curve (Validation)", fontsize=16, weight="bold")
plt.legend()
plt.grid(alpha=0.25)
plt.show()

# ---- Calibration Curve ----
prob_true, prob_pred = calibration_curve(y_val, p_val, n_bins=10, strategy="quantile")

plt.figure(figsize=(7, 6))
plt.plot(prob_pred, prob_true, linewidth=2.5, marker="o", color="#8E44AD")
plt.plot([0, 1], [0, 1], "--", color="#7f8c8d", linewidth=1.8)
plt.xlabel("Predicted Probability", fontsize=14)
plt.ylabel("Observed Frequency", fontsize=14)
plt.title("Calibration Curve (Validation)", fontsize=16, weight="bold")
plt.grid(alpha=0.25)
plt.show()

# ---- Print scores ----
print({
    "PR_AUC": float(average_precision_score(y_val, p_val)),
    "ROC_AUC": float(roc_auc_score(y_val, p_val)),
    "Brier": float(brier_score_loss(y_val, p_val))
})

{'PR_AUC': 0.3199754714071613, 'ROC_AUC': 0.8977654534180949, 'Brier': 0.12161006314446517}

# CALIBRATION
from sklearn.calibration import CalibratedClassifierCV

calibrated_lr = CalibratedClassifierCV(
    estimator=logreg_pipe.named_steps["clf"],  # not base_estimator
    cv="prefit",
    method="isotonic"
)
Z_val = logreg_pipe.named_steps["prep"].transform(X_val)
calibrated_lr.fit(Z_val, y_val)

def predict_calibrated(pipe, calib, X):
    Z = pipe.named_steps["prep"].transform(X)
    return calib.predict_proba(Z)[:, 1]

/usr/local/lib/python3.12/dist-packages/sklearn/calibration.py:333: UserWarning: The `cv='prefit'` option is deprecated in 1.6 and will be removed in 1.8. You can use CalibratedClassifierCV(FrozenEstimator(estimator)) instead.
  warnings.warn(

# Monthly aggregation for business KPI (Actual vs Predicted, MAE/MAPE)

def monthly_table(df_slice, p_hat, month_col="trial_month", target_col="label_120d"):
    d = df_slice.copy()
    d["p_hat"] = p_hat
    out = d.groupby(month_col).agg(
        actual   =(target_col, "sum"),
        predicted=("p_hat", "sum"),
        trials   =(target_col, "count")
    ).reset_index()
    out["mae"]  = (out["actual"] - out["predicted"]).abs()
    out["mape"] = out["mae"] / out["actual"].replace(0, np.nan)
    return out

# 1) Validation monthly KPIs – raw logistic
p_val = logreg_pipe.predict_proba(X_val)[:, 1]
val_monthly_lr = monthly_table(train.loc[is_val], p_val)
print("VAL monthly (LR raw):\n", val_monthly_lr, "\n",
      "MAE mean:", float(val_monthly_lr["mae"].mean()),
      "MAPE mean:", float(val_monthly_lr["mape"].mean()))

# 2) Validation monthly KPIs – calibrated logistic
p_val_cal = predict_calibrated(logreg_pipe, calibrated_lr, X_val)
val_monthly_lr_cal = monthly_table(train.loc[is_val], p_val_cal)
print("VAL monthly (LR calibrated):\n", val_monthly_lr_cal, "\n",
      "MAE mean:", float(val_monthly_lr_cal["mae"].mean()),
      "MAPE mean:", float(val_monthly_lr_cal["mape"].mean()))

# 3) Test monthly KPIs – raw and calibrated (same fitted logistic + calibration)
X_test = test[feature_cols].copy()

p_test = logreg_pipe.predict_proba(X_test)[:, 1]
test_monthly_lr = monthly_table(test, p_test)
print("TEST monthly (LR raw):\n", test_monthly_lr, "\n",
      "MAE mean:", float(test_monthly_lr["mae"].mean()),
      "MAPE mean:", float(test_monthly_lr["mape"].mean()))

p_test_cal = predict_calibrated(logreg_pipe, calibrated_lr, X_test)
test_monthly_lr_cal = monthly_table(test, p_test_cal)
print("TEST monthly (LR calibrated):\n", test_monthly_lr_cal, "\n",
      "MAE mean:", float(test_monthly_lr_cal["mae"].mean()),
      "MAPE mean:", float(test_monthly_lr_cal["mape"].mean()))

VAL monthly (LR raw):
   trial_month  actual   predicted  trials         mae      mape
0     2024-10     146  630.471072    2365  484.471072  3.318295
1     2024-11     126  610.321957    2512  484.321957  3.843825
2     2024-12     115  472.740440    2257  357.740440  3.110786 
 MAE mean: 442.1778228205899 MAPE mean: 3.424302166422654
VAL monthly (LR calibrated):
   trial_month  actual   predicted  trials        mae      mape
0     2024-10     146  151.268761    2365   5.268761  0.036087
1     2024-11     126  134.501415    2512   8.501415  0.067472
2     2024-12     115  101.229824    2257  13.770176  0.119741 
 MAE mean: 9.180117340246005 MAPE mean: 0.07443320430352669
TEST monthly (LR raw):
   trial_month  actual   predicted  trials         mae      mape
0     2025-01     147  584.970517    2496  437.970517  2.979391
1     2025-02     179  621.903042    2172  442.903042  2.474319
2     2025-03     158  607.223917    2230  449.223917  2.843189 
 MAE mean: 443.3658255327576 MAPE mean: 2.765633098138703
TEST monthly (LR calibrated):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  125.594742    2496  21.405258  0.145614
1     2025-02     179  137.199340    2172  41.800660  0.233523
2     2025-03     158  134.255182    2230  23.744818  0.150284 
 MAE mean: 28.983578530768913 MAPE mean: 0.17647363187729784

# Unified evaluator: trains pipe, returns metrics + monthly MAE/MAPE
from sklearn.base import clone

def evaluate_model(preprocess, clf, X_tr, y_tr, X_val, y_val, df_val_monthly, k=0.05, name="model"):
    """
    preprocess: ColumnTransformer (will be cloned)
    clf: sklearn classifier (will be cloned)
    df_val_monthly: the original validation slice (train.loc[is_val]) for monthly aggregation
    """
    pipe = Pipeline([("prep", clone(preprocess)), ("clf", clone(clf))])
    pipe.fit(X_tr, y_tr)
    p_val = pipe.predict_proba(X_val)[:,1]

    # classifier metrics
    pr_auc  = average_precision_score(y_val, p_val)
    roc_auc = roc_auc_score(y_val, p_val)
    brier   = brier_score_loss(y_val, p_val)

    # Precision@k / Recall@k
    cutoff = np.quantile(p_val, 1-k)
    topk   = (p_val >= cutoff).astype(int)
    prec_k = ((topk & (y_val==1)).sum() / max(topk.sum(),1))
    rec_k  = ((topk & (y_val==1)).sum() / max((y_val==1).sum(),1))

    # monthly KPIs
    val_monthly = monthly_table(df_val_monthly, p_val)
    mae_m = float(val_monthly["mae"].mean())
    mape_m = float(val_monthly["mape"].mean())

    return {
        "model": name,
        "PR_AUC": float(pr_auc),
        "ROC_AUC": float(roc_auc),
        "Brier": float(brier),
        f"Precision@{int(k*100)}%": float(prec_k),
        f"Recall@{int(k*100)}%": float(rec_k),
        "Monthly_MAE_mean": mae_m,
        "Monthly_MAPE_mean": mape_m,
        "pipe": pipe,            # return trained pipe (to plot PR/ROC for the winner)
        "p_val": p_val,          # to reuse for plots if needed
        "val_monthly_table": val_monthly
    }

# ============================================================
# Model race: RF + XGB from scratch, plus calibrated logistic
# ============================================================
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.base import clone
from sklearn.metrics import average_precision_score, roc_auc_score, brier_score_loss
import numpy as np
import pandas as pd

# Try to import XGBoost; if not installed
try:
    from xgboost import XGBClassifier
    has_xgb = True
except Exception:
    has_xgb = False
    print("NOTE: xgboost not installed; skipping XGBClassifier.")

# We let RF and  XGB be trained via evaluate_model
models = {
    "RandomForest": RandomForestClassifier(
        n_estimators=400,
        n_jobs=-1,
        class_weight="balanced_subsample",
        random_state=42,
    ),
}

if has_xgb:
    models["XGBoost"] = XGBClassifier(
        n_estimators=600,
        max_depth=6,
        learning_rate=0.05,
        subsample=0.8,
        colsample_bytree=0.8,
        objective="binary:logistic",
        eval_metric="logloss",
        n_jobs=-1,
        random_state=42,
    )

race_results = []
artifacts = {}
k = 0.05  # top-5% cutoff for Precision@k / Recall@k

# RF + XGB via the generic evaluator ---
for name, clf in models.items():
    res = evaluate_model(
        base_preprocess,
        clf,
        X_tr,
        y_tr,
        X_val,
        y_val,
        train.loc[is_val],
        k=k,
        name=name,
    )
    race_results.append(
        {k_: v for k_, v in res.items()
         if k_ not in {"pipe", "p_val", "val_monthly_table"}}
    )
    artifacts[name] = res

# Add calibrated logistic as an explicit row
# p_val_cal and val_monthly_lr_cal come from the calibration chunk
pr_auc_cal  = average_precision_score(y_val, p_val_cal)
roc_auc_cal = roc_auc_score(y_val, p_val_cal)
brier_cal   = brier_score_loss(y_val, p_val_cal)

# Precision@top-5% and Recall@top-5%
cutoff = np.quantile(p_val_cal, 1 - k)
topk   = (p_val_cal >= cutoff).astype(int)

tp  = ((topk == 1) & (y_val == 1)).sum()
prec_k = tp / max(topk.sum(), 1)
rec_k  = tp / max((y_val == 1).sum(), 1)

lr_cal_mae_mean  = float(val_monthly_lr_cal["mae"].mean())
lr_cal_mape_mean = float(val_monthly_lr_cal["mape"].mean())

race_results.append({
    "model": "LogReg_balanced (calibrated)",
    "PR_AUC": float(pr_auc_cal),
    "ROC_AUC": float(roc_auc_cal),
    "Brier": float(brier_cal),
    f"Precision@{int(k*100)}%": float(prec_k),
    f"Recall@{int(k*100)}%": float(rec_k),
    "Monthly_MAE_mean": lr_cal_mae_mean,
    "Monthly_MAPE_mean": lr_cal_mape_mean,
})

# Build the comparison table ---
comp = (
    pd.DataFrame(race_results)
      .sort_values("PR_AUC", ascending=False)
      .reset_index(drop=True)
)

print(comp)

                          model    PR_AUC   ROC_AUC     Brier  Precision@5%  \
0                       XGBoost  0.319277  0.903129  0.042703      0.355742   
1  LogReg_balanced (calibrated)  0.314559  0.901100  0.041848      0.296736   
2                  RandomForest  0.283013  0.883800  0.044060      0.299720   

   Recall@5%  Monthly_MAE_mean  Monthly_MAPE_mean  
0   0.328165          8.475866           0.070143  
1   0.516796          9.180117           0.074433  
2   0.276486          7.778377           0.061511

import pandas as pd
from IPython.display import Markdown, display

# comparison table for the current `comp`

metric_cols = [
    "PR_AUC",
    "ROC_AUC",
    "Brier",
    "Precision@5%",
    "Recall@5%",
    "Monthly_MAE_mean",
    "Monthly_MAPE_mean",
]

comp_plot = (
    comp.copy()
        .rename(columns={"model": "Model"})
        [["Model"] + metric_cols]
)

higher_is_better = ["PR_AUC", "ROC_AUC", "Precision@5%", "Recall@5%"]
lower_is_better  = ["Brier", "Monthly_MAE_mean", "Monthly_MAPE_mean"]

fmt_dict = {col: "{:.3f}" for col in metric_cols}

# Legend improved
legend_text = (
    "<span style='font-size:16px; font-weight:bold;'>Legend:</span><br>"
    "🟩 <b>Green (darker = better):</b> higher is better<br>"
    "🟥 <b>Red (darker = worse):</b> lower is better (errors)"
)
display(Markdown(legend_text))

# Styled table with bigger caption
comp_style = (
    comp_plot.style
        .format(fmt_dict)
        .background_gradient(subset=higher_is_better, cmap="Greens")
        .background_gradient(subset=lower_is_better, cmap="Reds")
        .set_caption("Performance Comparison Across Models")
        .set_table_styles([
            {
                "selector": "caption",
                "props": [
                    ("font-size", "20px"),
                    ("font-weight", "bold"),
                    ("text-align", "center"),
                    ("padding", "10px 0 10px 0")
                ]
            }
        ])
)

display(comp_style)

# XGBoost feature importance: permutation + SHAP on validation

from sklearn.inspection import permutation_importance
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Take the trained XGBoost pipeline from the model race
xgb_art = artifacts.get("XGBoost")
assert xgb_art is not None, "XGBoost artifacts not found – make sure the model race with XGBoost ran successfully."
xgb_pipe = xgb_art["pipe"]

# Permutation importance at *raw feature* level (easy to explain to marketing)
perm = permutation_importance(
    xgb_pipe,
    X_val, y_val,
    n_repeats=5,
    random_state=42,
    n_jobs=-1
)

perm_df = pd.DataFrame({
    "feature": feature_cols,
    "importance_mean": perm.importances_mean,
    "importance_std": perm.importances_std
}).sort_values("importance_mean", ascending=False)

print("Top-20 features by permutation importance (XGBoost):")
print(perm_df.head(20))

plt.figure(figsize=(8, 6))
top20 = perm_df.head(20).iloc[::-1]  # reverse for nicer horizontal plot
plt.barh(top20["feature"], top20["importance_mean"])
plt.xlabel("Permutation importance (mean decrease in score)")
plt.title("XGBoost – Top-20 features (permutation importance)")
plt.tight_layout()
plt.show()

Top-20 features by permutation importance (XGBoost):
                          feature  importance_mean  importance_std
26        distinct_segment_events         0.007345        0.000655
7            tech_employees_range         0.000392        0.000456
20               technician_added         0.000392        0.000105
13               agent_downloaded         0.000280        0.000217
10                business_region         0.000280        0.000366
32                    event_count         0.000252        0.000137
24                hubspot_clicked         0.000224        0.000228
12               ticket_generated         0.000168        0.000105
19               invoice_generate         0.000140        0.000089
17                     run_script         0.000140        0.000000
34                 pricing_intent         0.000084        0.000327
3                      ip_country         0.000084        0.000339
9             is_survey_completed         0.000028        0.000105
18               alerts_email_set         0.000000        0.000000
6              initial_atera_goal         0.000000        0.000000
8         previous_rmm_experience         0.000000        0.000000
14               report_generated        -0.000056        0.000112
2         num_of_installed_agents        -0.000140        0.000217
16  it_automation_profile_created        -0.000168        0.000056
31                     agents_bin        -0.000168        0.000186

# ----------------------------------------------
# SHAP Summary Plot
# ----------------------------------------------
import shap
import numpy as np
import matplotlib.pyplot as plt

# Initialize JS visualization (not needed for static plots, but no harm)
shap.initjs()

# Extract fitted parts
prep = xgb_pipe.named_steps["prep"]
xgb_clf = xgb_pipe.named_steps["clf"]

# Transform validation data
Z_val = prep.transform(X_val)

# Try to get feature names (OHE-expanded)
try:
    feature_names_tx = prep.get_feature_names_out()
except Exception:
    feature_names_tx = [f"f_{i}" for i in range(Z_val.shape[1])]

# SHAP explainer
explainer = shap.TreeExplainer(xgb_clf)

# Sample rows for speed
idx = np.random.choice(Z_val.shape[0], size=min(1000, Z_val.shape[0]), replace=False)
Z_sample = Z_val[idx]

# Compute SHAP values
shap_values = explainer(Z_sample)

# ----------------------------------------------
# Enhanced SHAP plot aesthetics
# ----------------------------------------------
plt.figure(figsize=(12, 8))

shap.summary_plot(
    shap_values,
    Z_sample,
    feature_names=feature_names_tx,
    max_display=20,                  # show only top 20 features
    plot_type="dot",                 # THE NICE DIAGONAL PLOT
    color=plt.get_cmap("viridis"),   # professional color map
    show=False
)

plt.title("Top-20 Feature Contributions Driving Conversion (SHAP)", fontsize=18, pad=20)
plt.xlabel("Impact on model output (positive → more likely to convert)", fontsize=14)
plt.ylabel("Features", fontsize=14)

# Improve tick font sizes
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)

plt.tight_layout()
plt.show()

from sklearn.metrics import average_precision_score
import numpy as np
import pandas as pd

# We reuse your calibrated logistic helper
# def predict_calibrated(pipe, calib, X):
#     Z = pipe.named_steps["prep"].transform(X)
#     return calib.predict_proba(Z)[:, 1]

def permutation_importance_logreg_cal(
    base_pipe,
    calibrator,
    X,
    y,
    feature_names,
    metric=average_precision_score,
    n_repeats=20,
    random_state=42,
):
    """
    Compute permutation importance for the *calibrated* logistic model.

    base_pipe   : fitted sklearn Pipeline (with 'prep' + logistic)
    calibrator  : fitted calibration object (e.g., IsotonicRegression)
    X, y        : validation data (features + labels)
    feature_names : list of feature column names to permute
    metric      : performance metric (default = PR-AUC)
    n_repeats   : how many shuffles per feature (stability)
    """
    rng = np.random.default_rng(random_state)

    # 1) Baseline performance on the original (unshuffled) data
    p_base = predict_calibrated(base_pipe, calibrator, X)
    base_score = metric(y, p_base)

    results = []

    # 2) Loop over features and measure how much the metric drops when shuffling each one
    for feat in feature_names:
        scores = []

        for r in range(n_repeats):
            # Copy the original X for each repeat to avoid accumulating noise
            X_perm = X.copy()

            # Shuffle this feature only
            shuffled = X_perm[feat].to_numpy().copy()
            rng.shuffle(shuffled)
            X_perm[feat] = shuffled

            # Recompute predictions and metric
            p_perm = predict_calibrated(base_pipe, calibrator, X_perm)
            scores.append(metric(y, p_perm))

        scores = np.array(scores)
        mean_score = scores.mean()
        std_score  = scores.std()

        # Importance = how much the performance *drops* when permuting this feature
        importance_mean = base_score - mean_score
        importance_std  = std_score

        results.append({
            "feature": feat,
            "importance_mean": float(importance_mean),
            "importance_std": float(importance_std),
        })

    imp_df = (
        pd.DataFrame(results)
          .sort_values("importance_mean", ascending=False)
          .reset_index(drop=True)
    )

    return imp_df, float(base_score)


# ==== Run permutation importance on the calibrated logistic (VALIDATION) ====
# Make sure X_val is a DataFrame with the same columns as in training.
X_val_features = X_val[feature_cols].copy()

logreg_cal_importance, logreg_cal_pr_auc = permutation_importance_logreg_cal(
    base_pipe=logreg_pipe,
    calibrator=calibrated_lr,
    X=X_val_features,
    y=y_val,
    feature_names=feature_cols,
    metric=average_precision_score,  # PR-AUC importance
    n_repeats=20,
)

print("Baseline PR-AUC (calibrated logistic, validation):", logreg_cal_pr_auc)
print("Top-20 features by permutation importance (LogReg calibrated):")
print(logreg_cal_importance.head(20))

# ------------------------------------------
#
# ------------------------------------------

plt.figure(figsize=(12, 9))

top20 = logreg_cal_importance.head(20).iloc[::-1]

bars = plt.barh(
    y=top20["feature"],
    width=top20["importance_mean"],
    color="#4A90E2",
    edgecolor="none"
)

# Labels & title
plt.xlabel(
    "Permutation importance (mean decrease in PR-AUC)",
    fontsize=19
)
plt.ylabel("")
plt.title(
    "Calibrated Logistic Regression – Top-20 Features\nPermutation Importance on Validation",
    fontsize=19,
    pad=18
)

# Axis ticks — slightly larger now
plt.yticks(fontsize=13)
plt.xticks(fontsize=13)

plt.grid(axis="x", linestyle="--", alpha=0.4)

plt.tight_layout()
plt.show()

Baseline PR-AUC (calibrated logistic, validation): 0.3145586050065097
Top-20 features by permutation importance (LogReg calibrated):
                        feature  importance_mean  importance_std
0                    agents_bin         0.043108        0.006220
1       registration_email_type         0.039384        0.007011
2                   campaign_id         0.038352        0.006275
3                pricing_intent         0.032436        0.004787
4          tech_employees_range         0.031597        0.005517
5                    ip_country         0.027462        0.005606
6                segment_events         0.026960        0.004258
7                  company_type         0.024533        0.009259
8                    channel_id         0.024343        0.004000
9               business_region         0.021936        0.003435
10  distinct_segment_page_views         0.021343        0.002109
11               agent_presence         0.015118        0.002769
12              clearbit_sector         0.013100        0.002144
13           has_business_email         0.012607        0.003979
14                  event_count         0.012326        0.001977
15          is_survey_completed         0.012208        0.004803
16      distinct_segment_events         0.011771        0.001833
17            clearbit_industry         0.011495        0.002650
18               email_x_agents         0.009728        0.002506
19   remote_connection__success         0.008412        0.001351

from sklearn.metrics import precision_recall_curve, roc_curve
from sklearn.calibration import calibration_curve
import matplotlib.pyplot as plt
import pandas as pd

# Take the chosen model from the model race (XGBoost)
winner = "XGBoost"
xgb_res  = artifacts[winner]
xgb_pipe = xgb_res["pipe"]          # fitted on (X_tr, y_tr) inside evaluate_model
p_val_xgb = xgb_res["p_val"]        # validation probabilities

print("Chosen model:", winner)

# PR and ROC curves on the validation window
prec, rec, thr = precision_recall_curve(y_val, p_val_xgb)
fpr,  tpr, _   = roc_curve(y_val, p_val_xgb)

plt.figure()
plt.plot(rec, prec)
plt.xlabel("Recall"); plt.ylabel("Precision")
plt.title(f"PR Curve (Validation) — {winner}")
plt.show()

plt.figure()
plt.plot(fpr, tpr)
plt.plot([0, 1], [0, 1], '--')
plt.xlabel("FPR"); plt.ylabel("TPR")
plt.title(f"ROC Curve (Validation) — {winner}")
plt.show()

# Calibration plot for XGBoost on validation
prob_true, prob_pred = calibration_curve(y_val, p_val_xgb, n_bins=10, strategy="quantile")
plt.figure()
plt.plot(prob_pred, prob_true)
plt.plot([0, 1], [0, 1], '--')
plt.xlabel("Predicted")
plt.ylabel("Observed")
plt.title(f"Calibration (Validation) — {winner}")
plt.show()

# Refit XGBoost on TRAIN+VAL and evaluate monthly on TEST
X_tr_final = pd.concat([X_tr, X_val], axis=0)
y_tr_final = pd.concat([y_tr, y_val], axis=0)

xgb_pipe.fit(X_tr_final, y_tr_final)

X_test = test[feature_cols].copy()
p_test_xgb = xgb_pipe.predict_proba(X_test)[:, 1]

test_monthly_xgb = monthly_table(test, p_test_xgb)
print("TEST monthly (XGBoost raw):\n", test_monthly_xgb)

Chosen model: XGBoost

TEST monthly (XGBoost raw):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  120.735725    2496  26.264275  0.178669
1     2025-02     179  134.668793    2172  44.331207  0.247660
2     2025-03     158  131.757980    2230  26.242020  0.166089

# Calibrate XGBoost probabilities and re-check monthly forecast

from sklearn.calibration import CalibratedClassifierCV
from sklearn.calibration import calibration_curve
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# Grab the pre-fitted XGBoost pipeline from the model race
xgb_art  = artifacts["XGBoost"]
xgb_pipe = xgb_art["pipe"]
p_val_xgb_raw = xgb_art["p_val"]          # raw validation probabilities
xgb_clf_prefit = xgb_pipe.named_steps["clf"]

# Fit an isotonic calibrator on the validation window (cv="prefit")
calibrated_xgb = CalibratedClassifierCV(
    estimator=xgb_clf_prefit,
    cv="prefit",
    method="isotonic"
)

Z_val_xgb = xgb_pipe.named_steps["prep"].transform(X_val)
calibrated_xgb.fit(Z_val_xgb, y_val)

def predict_calibrated_xgb(pipe, calib, X):
    Z = pipe.named_steps["prep"].transform(X)
    return calib.predict_proba(Z)[:, 1]

# Validation: monthly raw vs calibrated
val_monthly_xgb_raw = monthly_table(train.loc[is_val], p_val_xgb_raw)
print("VAL monthly (XGB raw):\n", val_monthly_xgb_raw, "\n",
      "MAE mean:", float(val_monthly_xgb_raw["mae"].mean()),
      "MAPE mean:", float(val_monthly_xgb_raw["mape"].mean()))

p_val_xgb_cal = predict_calibrated_xgb(xgb_pipe, calibrated_xgb, X_val)
val_monthly_xgb_cal = monthly_table(train.loc[is_val], p_val_xgb_cal)
print("VAL monthly (XGB calibrated):\n", val_monthly_xgb_cal, "\n",
      "MAE mean:", float(val_monthly_xgb_cal["mae"].mean()),
      "MAPE mean:", float(val_monthly_xgb_cal["mape"].mean()))

# Optional: calibration plot after calibration
prob_true_cal, prob_pred_cal = calibration_curve(
    y_val, p_val_xgb_cal, n_bins=10, strategy="quantile"
)
plt.figure()
plt.plot(prob_pred_cal, prob_true_cal)
plt.plot([0, 1], [0, 1], "--")
plt.xlabel("Predicted")
plt.ylabel("Observed")
plt.title("Calibration (Validation) — XGBoost calibrated")
plt.show()

# 4) Test: monthly raw vs calibrated
X_test = test[feature_cols].copy()

p_test_xgb_raw = xgb_pipe.predict_proba(X_test)[:, 1]
test_monthly_xgb_raw = monthly_table(test, p_test_xgb_raw)
print("TEST monthly (XGB raw):\n", test_monthly_xgb_raw, "\n",
      "MAE mean:", float(test_monthly_xgb_raw["mae"].mean()),
      "MAPE mean:", float(test_monthly_xgb_raw["mape"].mean()))

p_test_xgb_cal = predict_calibrated_xgb(xgb_pipe, calibrated_xgb, X_test)
test_monthly_xgb_cal = monthly_table(test, p_test_xgb_cal)
print("TEST monthly (XGB calibrated):\n", test_monthly_xgb_cal, "\n",
      "MAE mean:", float(test_monthly_xgb_cal["mae"].mean()),
      "MAPE mean:", float(test_monthly_xgb_cal["mape"].mean()))

VAL monthly (XGB raw):
   trial_month  actual   predicted  trials        mae      mape
0     2024-10     146  140.494614    2365   5.505386  0.037708
1     2024-11     126  125.318848    2512   0.681152  0.005406
2     2024-12     115   95.758942    2257  19.241058  0.167314 
 MAE mean: 8.475865681966146 MAPE mean: 0.07014254918354455

/usr/local/lib/python3.12/dist-packages/sklearn/calibration.py:333: UserWarning: The `cv='prefit'` option is deprecated in 1.6 and will be removed in 1.8. You can use CalibratedClassifierCV(FrozenEstimator(estimator)) instead.
  warnings.warn(

VAL monthly (XGB calibrated):
   trial_month  actual   predicted  trials       mae      mape
0     2024-10     146  152.446042    2365  6.446042  0.044151
1     2024-11     126  128.387128    2512  2.387128  0.018945
2     2024-12     115  106.167257    2257  8.832743  0.076806 
 MAE mean: 5.888637675438076 MAPE mean: 0.04663429780582073

TEST monthly (XGB raw):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  120.735725    2496  26.264275  0.178669
1     2025-02     179  134.668793    2172  44.331207  0.247660
2     2025-03     158  131.757980    2230  26.242020  0.166089 
 MAE mean: 32.27916717529297 MAPE mean: 0.19747254749927434
TEST monthly (XGB calibrated):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  116.735122    2496  30.264878  0.205884
1     2025-02     179  131.280200    2172  47.719800  0.266591
2     2025-03     158  129.766261    2230  28.233739  0.178695 
 MAE mean: 35.40613902474676 MAPE mean: 0.21705637889402787

# XGBoost (raw & calibrated) vs baseline: metrics + monthly forecast

from sklearn.metrics import (
    average_precision_score,
    roc_auc_score,
    brier_score_loss,
    precision_recall_curve
)
import numpy as np

def summarize_model(y_true, p, k=0.05, label="model"):
    """Print standard classifier metrics + Precision/Recall@Top-k%."""
    pr_auc  = average_precision_score(y_true, p)
    roc_auc = roc_auc_score(y_true, p)
    brier   = brier_score_loss(y_true, p)

    # Precision/Recall@Top-k% (e.g. top-5%)
    cutoff = np.quantile(p, 1 - k)
    topk   = (p >= cutoff).astype(int)
    tp     = ((topk == 1) & (y_true == 1)).sum()
    prec_k = tp / max(topk.sum(), 1)
    rec_k  = tp / max((y_true == 1).sum(), 1)

    print(f"\n=== {label} ===")
    print({
        "PR_AUC": float(pr_auc),
        "ROC_AUC": float(roc_auc),
        "Brier": float(brier),
        f"Precision@{int(k*100)}%": float(prec_k),
        f"Recall@{int(k*100)}%": float(rec_k),
    })


# ---------- VALIDATION METRICS ----------

# XGBoost raw / calibrated on validation
summarize_model(y_val.values, p_val_xgb_raw, k=0.05, label="XGB_raw (VAL)")
summarize_model(y_val.values, p_val_xgb_cal, k=0.05, label="XGB_calibrated (VAL)")

# Baseline heuristic on validation (0/1 scores)
p_val_baseline = train.loc[is_val, "baseline_pred"].astype(float).values
summarize_model(y_val.values, p_val_baseline, k=0.05, label="Baseline rule (VAL)")

# Monthly VAL: XGB calibrated vs baseline
val_monthly_xgb_cal = monthly_table(train.loc[is_val], p_val_xgb_cal)
val_monthly_base    = monthly_table(train.loc[is_val], p_val_baseline)

print("\nVAL monthly (XGB calibrated):\n", val_monthly_xgb_cal,
      "\n MAE mean:", float(val_monthly_xgb_cal["mae"].mean()),
      "MAPE mean:", float(val_monthly_xgb_cal["mape"].mean()))

print("\nVAL monthly (Baseline rule):\n", val_monthly_base,
      "\n MAE mean:", float(val_monthly_base["mae"].mean()),
      "MAPE mean:", float(val_monthly_base["mape"].mean()))


# ---------- TEST METRICS ----------

# XGBoost raw / calibrated on TEST (probabilities כבר חישבנו קודם)
summarize_model(test["label_120d"].values, p_test_xgb_raw, k=0.05, label="XGB_raw (TEST)")
summarize_model(test["label_120d"].values, p_test_xgb_cal, k=0.05, label="XGB_calibrated (TEST)")

# Baseline heuristic on TEST
p_test_baseline = test["baseline_pred"].astype(float).values
summarize_model(test["label_120d"].values, p_test_baseline, k=0.05, label="Baseline rule (TEST)")

# Monthly TEST: XGB calibrated vs baseline
test_monthly_xgb_cal = monthly_table(test, p_test_xgb_cal)
test_monthly_base    = monthly_table(test, p_test_baseline)

print("\nTEST monthly (XGB calibrated):\n", test_monthly_xgb_cal,
      "\n MAE mean:", float(test_monthly_xgb_cal["mae"].mean()),
      "MAPE mean:", float(test_monthly_xgb_cal["mape"].mean()))

print("\nTEST monthly (Baseline rule):\n", test_monthly_base,
      "\n MAE mean:", float(test_monthly_base["mae"].mean()),
      "MAPE mean:", float(test_monthly_base["mape"].mean()))

=== XGB_raw (VAL) ===
{'PR_AUC': 0.31927728217171886, 'ROC_AUC': 0.9031291158593217, 'Brier': 0.042703368447574205, 'Precision@5%': 0.3557422969187675, 'Recall@5%': 0.3281653746770026}

=== XGB_calibrated (VAL) ===
{'PR_AUC': 0.640522479670993, 'ROC_AUC': 0.9601913224711989, 'Brier': 0.028924154612450043, 'Precision@5%': 0.6049723756906077, 'Recall@5%': 0.5658914728682171}

=== Baseline rule (VAL) ===
{'PR_AUC': 0.13153888658347962, 'ROC_AUC': 0.7401067140951534, 'Brier': 0.1944210821418559, 'Precision@5%': 0.17018469656992086, 'Recall@5%': 0.6666666666666666}

VAL monthly (XGB calibrated):
   trial_month  actual   predicted  trials       mae      mape
0     2024-10     146  152.446042    2365  6.446042  0.044151
1     2024-11     126  128.387128    2512  2.387128  0.018945
2     2024-12     115  106.167257    2257  8.832743  0.076806 
 MAE mean: 5.888637675438076 MAPE mean: 0.04663429780582073

VAL monthly (Baseline rule):
   trial_month  actual  predicted  trials    mae      mape
0     2024-10     146      551.0    2365  405.0  2.773973
1     2024-11     126      517.0    2512  391.0  3.103175
2     2024-12     115      448.0    2257  333.0  2.895652 
 MAE mean: 376.3333333333333 MAPE mean: 2.924266459942458

=== XGB_raw (TEST) ===
{'PR_AUC': 0.374085292346017, 'ROC_AUC': 0.8908964313601188, 'Brier': 0.052713943890637265, 'Precision@5%': 0.4318840579710145, 'Recall@5%': 0.30785123966942146}

=== XGB_calibrated (TEST) ===
{'PR_AUC': 0.3549690088425562, 'ROC_AUC': 0.8891237723780882, 'Brier': 0.05501084968169637, 'Precision@5%': 0.4275, 'Recall@5%': 0.35330578512396693}

=== Baseline rule (TEST) ===
{'PR_AUC': 0.15614457850034597, 'ROC_AUC': 0.7337210441003281, 'Brier': 0.214554943461873, 'Precision@5%': 0.19781553398058252, 'Recall@5%': 0.6735537190082644}

TEST monthly (XGB calibrated):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  116.735122    2496  30.264878  0.205884
1     2025-02     179  131.280200    2172  47.719800  0.266591
2     2025-03     158  129.766261    2230  28.233739  0.178695 
 MAE mean: 35.40613902474676 MAPE mean: 0.21705637889402787

TEST monthly (Baseline rule):
   trial_month  actual  predicted  trials    mae      mape
0     2025-01     147      531.0    2496  384.0  2.612245
1     2025-02     179      590.0    2172  411.0  2.296089
2     2025-03     158      527.0    2230  369.0  2.335443 
 MAE mean: 388.0 MAPE mean: 2.4145924404695758

import matplotlib.pyplot as plt
import numpy as np

# -----------------------------------------------
# TEST metrics — only calibrated vs baseline
# -----------------------------------------------
test_data = {
    "XGB Calibrated": [0.3549690088425562, 0.8891237723780882, 0.4275, 0.35330578512396693],
    "Baseline Rule":  [0.15614457850034597, 0.7337210441003281, 0.19781553398058252, 0.6735537190082644]
}

metric_names = ["PR-AUC", "ROC-AUC", "Precision@5%", "Recall@5%"]
models = list(test_data.keys())

# Convert to array
data = np.array([test_data[m] for m in models])

# -----------------------------------------------
# Plot TEST comparison
# -----------------------------------------------
plt.figure(figsize=(10,6))
x = np.arange(len(metric_names))
width = 0.35

plt.bar(x - width/2, data[0], width, label="XGB Calibrated", color="#50E3C2")
plt.bar(x + width/2, data[1], width, label="Baseline Rule", color="#9B9B9B")

plt.xticks(x, metric_names, fontsize=13)
plt.ylabel("Score (0–1)", fontsize=14)
plt.title("Model Comparison – TEST (Calibrated XGB vs Baseline)", fontsize=16, pad=10)

plt.grid(axis="y", linestyle="--", alpha=0.35)
plt.legend(fontsize=12)

# Add percentage labels above bars
for i in range(data.shape[0]):
    for j in range(len(metric_names)):
        plt.text(
            x[j] + (i-0.5)*width,
            data[i, j] + 0.01,
            f"{data[i,j]*100:.1f}%",
            ha="center",
            fontsize=11
        )

plt.tight_layout()
plt.show()

import matplotlib.pyplot as plt

# -----------------------------
# Data
# -----------------------------
months = ["2025-01", "2025-02", "2025-03"]

actual      = [147, 179, 158]
xgb_pred    = [116.735122, 131.280200, 129.766261]
base_pred   = [531.0,       590.0,       527.0]

MAPE_xgb  = 0.217   # ~21.7%
MAPE_base = 2.414   # ~241.4%

# -----------------------------
# Plot
# -----------------------------
plt.figure(figsize=(12,7))

# Actual
plt.plot(months, actual, marker="o", linewidth=3, label="Actual",
         color="#1B84E7")

# XGB calibrated
plt.plot(months, xgb_pred, marker="o", linewidth=3, label=f"XGB Calibrated  (MAPE: {MAPE_xgb*100:.1f}%)",
         color="#50E3C2")

# Baseline
plt.plot(months, base_pred, marker="o", linewidth=3, label=f"Baseline Rule  (MAPE: {MAPE_base*100:.1f}%)",
         color="#9B9B9B")

# Title + labels
plt.title("Monthly Conversions – Actual vs Predicted (TEST)",
          fontsize=18, pad=15)

plt.xlabel("Month", fontsize=14)
plt.ylabel("Conversions", fontsize=14)

# Grid
plt.grid(alpha=0.3, linestyle="--")

# Legend
plt.legend(fontsize=13, frameon=True, edgecolor="#DDDDDD")

# Tighter layout
plt.tight_layout()
plt.show()

from sklearn.metrics import confusion_matrix, classification_report
import numpy as np

# Calibrated probabilities for validation and test
p_val_xgb_cal = predict_calibrated_xgb(xgb_pipe, calibrated_xgb, X_val)
X_test = test[feature_cols].copy()
p_test_xgb_cal = predict_calibrated_xgb(xgb_pipe, calibrated_xgb, X_test)

def confusion_at_topk(y_true, p_hat, k=0.05):
    """
    Build a confusion matrix at a Top-k% operating point.
    We choose the cutoff so that about k of the users are predicted as positive.
    """
    cutoff = np.quantile(p_hat, 1 - k)
    y_pred = (p_hat >= cutoff).astype(int)
    cm = confusion_matrix(y_true, y_pred)
    return cutoff, y_pred, cm

# Validation
cut_val, y_val_pred, cm_val = confusion_at_topk(y_val, p_val_xgb_cal, k=0.05)
print(f"Validation cutoff (Top-5%): {cut_val:.4f}")
print("Confusion matrix – validation (rows = true, cols = predicted):")
print(cm_val)
print("\nClassification report – validation:")
print(classification_report(y_val, y_val_pred, digits=3))

# Test
cut_test, y_test_pred, cm_test = confusion_at_topk(test["label_120d"].astype(int),
                                                   p_test_xgb_cal, k=0.05)
print(f"\nTest cutoff (Top-5%): {cut_test:.4f}")
print("Confusion matrix – test (rows = true, cols = predicted):")
print(cm_test)
print("\nClassification report – test:")
print(classification_report(test["label_120d"].astype(int), y_test_pred, digits=3))

Validation cutoff (Top-5%): 0.3711
Confusion matrix – validation (rows = true, cols = predicted):
[[6604  143]
 [ 168  219]]

Classification report – validation:
              precision    recall  f1-score   support

           0      0.975     0.979     0.977      6747
           1      0.605     0.566     0.585       387

    accuracy                          0.956      7134
   macro avg      0.790     0.772     0.781      7134
weighted avg      0.955     0.956     0.956      7134


Test cutoff (Top-5%): 0.3043
Confusion matrix – test (rows = true, cols = predicted):
[[6185  229]
 [ 313  171]]

Classification report – test:
              precision    recall  f1-score   support

           0      0.952     0.964     0.958      6414
           1      0.427     0.353     0.387       484

    accuracy                          0.921      6898
   macro avg      0.690     0.659     0.672      6898
weighted avg      0.915     0.921     0.918      6898

import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix

# ----------------------------------------------------
# Helper: compute predictions at a Top-k% operating point
# ----------------------------------------------------
def preds_at_topk(p_hat, k=0.05):
    """
    Assign class labels using a Top-k% cutoff.
    We sort predicted probabilities and label the top k% as '1'.
    """
    cutoff = np.quantile(p_hat, 1 - k)
    y_pred = (p_hat >= cutoff).astype(int)
    return cutoff, y_pred


# ----------------------------------------------------
# Load TEST true labels
# ----------------------------------------------------
y_test = test["label_120d"].astype(int).values



# ----------------------------------------------------
# Predictions (Top 5%)
# ----------------------------------------------------
cut_xgb,  y_pred_xgb  = preds_at_topk(p_test_xgb_cal,  k=0.05)
cut_base, y_pred_base = preds_at_topk(p_test_baseline, k=0.05)

cm_xgb  = confusion_matrix(y_test, y_pred_xgb)
cm_base = confusion_matrix(y_test, y_pred_base)

print("XGB cutoff:", cut_xgb)
print("Baseline cutoff:", cut_base)
print("XGB CM:\n", cm_xgb)
print("Baseline CM:\n", cm_base)


# ----------------------------------------------------
# Color scheme: 4 intuitive colors for marketing slides
# ----------------------------------------------------
def get_cell_color(i, j):
    """
    Returns a color for each cell in the confusion matrix:
      - Green: True Positive
      - Blue-gray: True Negative
      - Orange: False Positive
      - Red: False Negative
    i = true label, j = predicted label
    """
    if i == 1 and j == 1:
        return "#4CAF50"   # green (TP)
    if i == 0 and j == 0:
        return "#B0BEC5"   # blue-gray (TN)
    if i == 0 and j == 1:
        return "#FFB74D"   # orange (FP)
    if i == 1 and j == 0:
        return "#E57373"   # red (FN)


# ----------------------------------------------------
#  confusion matrix
# ----------------------------------------------------
def plot_cm_custom(cm, title, ax):
    """
    Draws a confusion matrix with:
      - Custom colors per error type
      - Counts + percentages
      - Clean labeling for slides
    """
    total = cm.sum()
    cm_pct = cm / total

    # Axis setup
    ax.set_xticks([0,1])
    ax.set_yticks([0,1])
    ax.set_xticklabels(["No convert", "Convert"], fontsize=11)
    ax.set_yticklabels(["No convert", "Convert"], fontsize=11)

    ax.set_xlabel("Predicted label", fontsize=11)
    ax.set_ylabel("True label", fontsize=11)
    ax.set_title(title, fontsize=14, pad=10)

    # Draw colored rectangles + text
    for i in range(2):
        for j in range(2):
            ax.add_patch(plt.Rectangle(
                (j - 0.5, i - 0.5), 1, 1,
                facecolor=get_cell_color(i, j),
                edgecolor="white",
                linewidth=2
            ))
            ax.text(
                j, i,
                f"{cm[i, j]:,}\n({cm_pct[i, j]*100:.1f}%)",
                ha="center", va="center",
                fontsize=12, color="black", fontweight="bold"
            )

    # Fix axis limits
    ax.set_xlim(-0.5, 1.5)
    ax.set_ylim(1.5, -0.5)


# ----------------------------------------------------
# Plot: XGB calibrated vs Baseline (TEST only)
# ----------------------------------------------------
fig, axes = plt.subplots(1, 2, figsize=(12, 5))

plot_cm_custom(
    cm_xgb,
    "Calibrated XGB – TEST (Top 5%)",
    axes[0]
)

plot_cm_custom(
    cm_base,
    "Baseline Rule – TEST (Top 5%)",
    axes[1]
)

plt.tight_layout()
plt.show()

XGB cutoff: 0.30434781312942505
Baseline cutoff: 1.0
XGB CM:
 [[6185  229]
 [ 313  171]]
Baseline CM:
 [[5092 1322]
 [ 158  326]]

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import precision_score, recall_score

y_test = test["label_120d"].astype(int).values
p_test = p_test_xgb_cal  # calibrated XGBoost probabilities on TEST

def metrics_at_topk(y_true, p_hat, k=0.05):
    """
    Compute precision/recall when we take the top-k% users
    (according to the score) as 'hot leads'.
    """
    cutoff = np.quantile(p_hat, 1 - k)
    y_pred = (p_hat >= cutoff).astype(int)
    prec = precision_score(y_true, y_pred)
    rec  = recall_score(y_true, y_pred)
    coverage = y_pred.mean()
    return cutoff, prec, rec, coverage

# Table for Top-5% and Top-20%

ks = [0.05, 0.20]
rows = []
for k in ks:
    cutoff, prec, rec, cov = metrics_at_topk(y_test, p_test, k)
    rows.append({
        "top_%": int(k * 100),
        "cutoff": cutoff,
        "precision": prec,
        "recall": rec,
        "coverage": cov
    })

summary_k = pd.DataFrame(rows)
summary_k_display = summary_k.copy()
for col in ["precision", "recall", "coverage"]:
    summary_k_display[col] = (summary_k_display[col] * 100).round(1)

print("Precision / Recall / Coverage on TEST:")
print(summary_k_display)

Precision / Recall / Coverage on TEST:
   top_%    cutoff  precision  recall  coverage
0      5  0.304348       42.8    35.3       5.8
1     20  0.047619       23.4    83.5      25.0

plt.style.use("default")
k_grid = np.linspace(0.01, 0.30, 30)

prec_list = []
rec_list = []
cov_list = []

for k in k_grid:
    _, prec, rec, cov = metrics_at_topk(y_test, p_test, k)
    prec_list.append(prec)
    rec_list.append(rec)
    cov_list.append(cov)

# Convert to percentages for a presentation-friendly plot
k_grid_pct = k_grid * 100
prec_pct = np.array(prec_list) * 100
rec_pct  = np.array(rec_list) * 100

plt.figure(figsize=(7,4))
plt.plot(k_grid_pct, prec_pct, marker="o", label="Precision (% of hot leads that buy)")
plt.plot(k_grid_pct, rec_pct,  marker="o", label="Recall (% of all buyers we catch)")
plt.xlabel("% of trials flagged as 'hot'")
plt.ylabel("Percentage")
plt.title("Precision / Recall vs % of trials contacted (TEST – calibrated XGBoost)")
plt.ylim(0, 100)
plt.grid(True, alpha=0.3)

# Highlighting the 5% and 20% points
for k_special in [5, 20]:
    idx = np.argmin(np.abs(k_grid_pct - k_special))
    plt.scatter([k_grid_pct[idx]], [prec_pct[idx]], s=60)
    plt.scatter([k_grid_pct[idx]], [rec_pct[idx]], s=60)

plt.legend()
plt.tight_layout()
plt.show()

import numpy as np
import pandas as pd


def campaign_table(df,
                   p_hat,
                   campaign_col="campaign_id",
                   target_col="label_120d",
                   n_top=25,
                   min_trials=50):
    """
    Build a per-campaign performance table for a given model.

    Parameters
    ----------
    df : pd.DataFrame
        Slice of data (e.g., TEST) with campaign and label columns.
    p_hat : array-like, shape (n_samples,)
        Calibrated predicted probabilities for df.
    campaign_col : str
        Column name of the marketing campaign ID.
    target_col : str
        Column name of the binary label (0/1 = no/yes conversion).
    n_top : int
        Number of top campaigns to keep (by number of trials).
    min_trials : int
        Minimum number of trials per campaign to keep it in the table
        (filters out tiny campaigns with very few trials).

    Returns
    -------
    grp_top : pd.DataFrame
        Table with trials, actual conversions, model-predicted conversions
        and MAE/MAPE per campaign, for the top campaigns.
    """
    tmp = df.copy()
    tmp["p_hat"] = p_hat

    # Aggregate per campaign: how many trials, how many real conversions,
    # and how many conversions the model predicts (sum of probabilities).
    grp = (
        tmp
        .groupby(campaign_col)
        .agg(
            trials   =(target_col, "count"),
            actual   =(target_col, "sum"),
            pred_model=("p_hat", "sum")
        )
        .reset_index()
    )

    # Absolute and relative error per campaign (for the model)
    grp["mae_model"]  = (grp["actual"] - grp["pred_model"]).abs()
    grp["mape_model"] = grp["mae_model"] / grp["actual"].replace(0, np.nan)

    # Keep only campaigns with enough volume
    grp = grp[grp["trials"] >= min_trials]

    # Take the n_top largest campaigns by number of trials
    grp_top = grp.sort_values("trials", ascending=False).head(n_top)

    return grp_top


# Get calibrated XGBoost probabilities on TEST --- #

X_test = test[feature_cols].copy()
p_test_xgb_cal = predict_calibrated_xgb(xgb_pipe, calibrated_xgb, X_test)

# Build Top-25 campaigns table

top25_xgb = campaign_table(
    test,
    p_test_xgb_cal,
    campaign_col="campaign_id",
    target_col="label_120d",
    n_top=25,
    min_trials=50
)

print("Top-25 campaigns – calibrated XGBoost (TEST):")
print(top25_xgb.head(25).to_string(index=False))

Top-25 campaigns – calibrated XGBoost (TEST):
 campaign_id  trials  actual  pred_model  mae_model  mape_model
         218    2154     207  165.872281  41.127719    0.198685
         547     469       1    1.651815   0.651815    0.651815
         406     338      15    3.794076  11.205924    0.747062
         256     211      27   23.817771   3.182229    0.117860
         202     154       2    2.253323   0.253323    0.126661
          19     138       8    4.786589   3.213411    0.401676
         268     133       2    2.115023   0.115023    0.057512
         156     117       3    1.720864   1.279136    0.426379
         473     113       4    2.068092   1.931908    0.482977
         429     111       1    0.368973   0.631027    0.631027
         146      86       0    0.505538   0.505538         NaN
         548      84      22   13.650317   8.349683    0.379531
           6      83       3    3.505080   0.505080    0.168360
         171      77       0    0.809940   0.809940         NaN
         380      73       0    0.490677   0.490677         NaN
          41      68       1    0.758312   0.241688    0.241688
         558      68      10   13.688219   3.688219    0.368822
         333      67       6    7.149170   1.149170    0.191528
         511      67       5    2.312919   2.687081    0.537416
         293      65      11    5.633362   5.366638    0.487876
          93      65       3    3.708483   0.708483    0.236161
         409      63       2    2.248141   0.248141    0.124071
          55      62       1    0.070536   0.929464    0.929464
         253      59       0    0.019092   0.019092         NaN
         534      58       3    1.220322   1.779678    0.593226

import numpy as np
import matplotlib.pyplot as plt

# Use all Top-25 campaigns (already aggregated in top25_xgb)
plot_df = top25_xgb.sort_values("trials", ascending=False).head(25)

x = np.arange(len(plot_df))
width = 0.35

plt.figure(figsize=(14, 7))

# Bars: actual vs model forecast
plt.bar(x - width/2,
        plot_df["actual"],
        width,
        label="Actual conversions")

plt.bar(x + width/2,
        plot_df["pred_model"],
        width,
        label="Model forecast (calibrated XGBoost)")

# X-axis: campaign IDs
plt.xticks(x,
           plot_df["campaign_id"].astype(str),
           rotation=60,
           ha="right")

plt.ylabel("Number of conversions")
plt.title("Top-25 campaigns – Actual vs Model forecast (TEST, calibrated XGBoost)")
plt.legend()
plt.grid(axis="y", alpha=0.3)
plt.tight_layout()
plt.show()

import matplotlib.pyplot as plt
import numpy as np

df = top25_xgb.sort_values("actual", ascending=True).tail(25)

plt.figure(figsize=(12, 10))

y_pos = np.arange(len(df))

plt.barh(y_pos - 0.2,
         df["actual"],
         height=0.4,
         label="Actual conversions",
         color="#1B84E7")

plt.barh(y_pos + 0.2,
         df["pred_model"],
         height=0.4,
         label="Model forecast (XGBoost)",
         color="#13C4A3")

plt.yticks(y_pos,
           df["campaign_id"].astype(str),
           fontsize=12)

plt.xlabel("Conversions", fontsize=13)
plt.title("Top-25 Campaigns – Actual vs Forecast (TEST, XGBoost Calibrated)",
          fontsize=15,
          pad=15)

plt.legend(fontsize=12)
plt.grid(axis="x", alpha=0.3)

plt.tight_layout()
plt.show()

# Baseline rule:
# 1 if (business email) AND (at least 1 installed agent), else 0.
def baseline_score(df):
    """
    Implement the legacy marketing heuristic:
    'Business email + 1+ installed agents' -> hot (=1), otherwise 0.
    """
    return (
        (df["has_business_email"] == 1) &
        (df["agent_presence"] == 1)
    ).astype(int)


p_test_baseline = baseline_score(test)


def add_baseline_to_campaigns(df,
                              p_model,
                              p_base,
                              campaign_col="campaign_id",
                              target_col="label_120d",
                              n_top=25,
                              min_trials=50):
    """
    Build a per-campaign comparison between the model and the baseline rule.

    Parameters
    ----------
    df : pd.DataFrame
        Slice of data (e.g., TEST) with campaign and label columns.
    p_model : array-like
        Model predicted probabilities (calibrated XGBoost) for df.
    p_base : array-like
        Baseline "scores" for df (0/1 from the rule).
    campaign_col : str
        Campaign column name.
    target_col : str
        Binary label column name.
    n_top : int
        Number of top campaigns (by number of trials) to present.
    min_trials : int
        Minimum number of trials per campaign to keep.

    Returns
    -------
    grp_top : pd.DataFrame
        Table with, for each campaign:
        - trials, actual conversions
        - predicted conversions by model and by baseline
        - MAE/MAPE for both model and baseline.
    """
    tmp = df.copy()
    tmp["p_model"] = p_model
    tmp["p_base"]  = p_base

    grp = (
        tmp
        .groupby(campaign_col)
        .agg(
            trials      =(target_col, "count"),
            actual      =(target_col, "sum"),
            pred_model  =("p_model", "sum"),
            pred_base   =("p_base",  "sum")
        )
        .reset_index()
    )

    # Absolute error per campaign
    grp["mae_model"] = (grp["actual"] - grp["pred_model"]).abs()
    grp["mae_base"]  = (grp["actual"] - grp["pred_base"]).abs()

    # Relative error per campaign (MAPE)
    grp["mape_model"] = grp["mae_model"] / grp["actual"].replace(0, np.nan)
    grp["mape_base"]  = grp["mae_base"]  / grp["actual"].replace(0, np.nan)

    # Filter out very small campaigns
    grp = grp[grp["trials"] >= min_trials]

    # Top campaigns by number of trials
    grp_top = grp.sort_values("trials", ascending=False).head(n_top)

    return grp_top


top25_cmp = add_baseline_to_campaigns(
    test,
    p_test_xgb_cal,
    p_test_baseline,
    campaign_col="campaign_id",
    target_col="label_120d",
    n_top=25,
    min_trials=50
)

print("Top-25 campaigns – model vs baseline (TEST):")
print(top25_cmp.to_string(index=False))

Top-25 campaigns – model vs baseline (TEST):
 campaign_id  trials  actual  pred_model  pred_base  mae_model  mae_base  mape_model  mape_base
         218    2154     207  165.872281        593  41.127719       386    0.198685   1.864734
         547     469       1    1.651815         59   0.651815        58    0.651815  58.000000
         406     338      15    3.794076         26  11.205924        11    0.747062   0.733333
         256     211      27   23.817771         74   3.182229        47    0.117860   1.740741
         202     154       2    2.253323         11   0.253323         9    0.126661   4.500000
          19     138       8    4.786589         33   3.213411        25    0.401676   3.125000
         268     133       2    2.115023         31   0.115023        29    0.057512  14.500000
         156     117       3    1.720864         21   1.279136        18    0.426379   6.000000
         473     113       4    2.068092         22   1.931908        18    0.482977   4.500000
         429     111       1    0.368973          8   0.631027         7    0.631027   7.000000
         146      86       0    0.505538          2   0.505538         2         NaN        NaN
         548      84      22   13.650317         31   8.349683         9    0.379531   0.409091
           6      83       3    3.505080         13   0.505080        10    0.168360   3.333333
         171      77       0    0.809940         12   0.809940        12         NaN        NaN
         380      73       0    0.490677          7   0.490677         7         NaN        NaN
          41      68       1    0.758312          9   0.241688         8    0.241688   8.000000
         558      68      10   13.688219         25   3.688219        15    0.368822   1.500000
         333      67       6    7.149170         30   1.149170        24    0.191528   4.000000
         511      67       5    2.312919         20   2.687081        15    0.537416   3.000000
         293      65      11    5.633362         28   5.366638        17    0.487876   1.545455
          93      65       3    3.708483         23   0.708483        20    0.236161   6.666667
         409      63       2    2.248141         15   0.248141        13    0.124071   6.500000
          55      62       1    0.070536          6   0.929464         5    0.929464   5.000000
         253      59       0    0.019092          2   0.019092         2         NaN        NaN
         534      58       3    1.220322          3   1.779678         0    0.593226   0.000000

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np

# Start from the comparison table `top25_cmp`
# Keep only the 25 largest campaigns by trials
heat_df = (
    top25_cmp
      .sort_values("trials", ascending=False)
      .head(25)
      .copy()
)

# Keep campaigns with at least one actual conversion
heat_df = heat_df[heat_df["actual"] > 0].copy()

# Compute MAPE (%) for model and baseline
heat_df["Model MAPE (%)"]    = (heat_df["mae_model"] / heat_df["actual"]) * 100
heat_df["Baseline MAPE (%)"] = (heat_df["mae_base"]  / heat_df["actual"]) * 100

# Long → wide format for the heatmap
melted = heat_df.melt(
    id_vars="campaign_id",
    value_vars=["Model MAPE (%)", "Baseline MAPE (%)"],
    var_name="Method",
    value_name="MAPE_%"
)

pivot = melted.pivot(index="campaign_id", columns="Method", values="MAPE_%")

sns.set_theme(style="white")
sns.set_context("talk")

# Robust color scale: ignore extreme outliers (e.g. 5800%)
vals = pivot.to_numpy().astype(float)
vmin = 0
vmax = np.percentile(vals, 95)   # you can change 95 → 90/98 to tune

plt.figure(figsize=(8, 7))
ax = sns.heatmap(
    pivot,
    annot=True,
    fmt=".1f",
    cmap="viridis_r",             # same palette as before, but with vmin/vmax
    vmin=vmin,
    vmax=vmax,
    linewidths=0.6,               # horizontal/vertical grid lines
    linecolor="white",
    cbar_kws={"label": "MAPE (%)"}
)

ax.set_xlabel("")
ax.set_ylabel("Campaign ID")
ax.set_title("Forecast error by campaign – Model vs Baseline (TEST)", pad=20)

# Clean axis labels
ax.set_xticklabels(ax.get_xticklabels(), rotation=0)
ax.set_yticklabels(ax.get_yticklabels(), rotation=0)

plt.tight_layout()
plt.show()

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# ============================================
# Build monthly EDA table (train + test)
# ============================================

# If you don't have `val` as a separate split, just concatenate train + test
df_all = pd.concat([train, test], ignore_index=True)

# Ensure trial_month is a string / period-like column suitable for grouping
# (If it's already YYYY-MM string or period, this will still be fine)
df_all["trial_month"] = df_all["trial_month"].astype(str)

monthly_eda = (
    df_all
      .groupby("trial_month", as_index=False)
      .agg(
          actual_conversions=("label_120d", "sum"),
          trials=("label_120d", "size")
      )
      .sort_values("trial_month")
)

# Keep only the last 12 months for the EDA chart
monthly_eda_recent = monthly_eda.tail(12).reset_index(drop=True)

# ============================================
# Plot: actual conversions per month (last 12 months)
# ============================================

sns.set_theme(style="whitegrid", context="talk")

fig, ax = plt.subplots(figsize=(11, 5))

# Dashboard-style background
fig.patch.set_facecolor("#f7f7fa")
ax.set_facecolor("#f9fafb")

# X positions
x = np.arange(len(monthly_eda_recent))

# Strong light-blue color for bars
bar_color = "#0ea5e9"      # cyan / strong light blue
edge_color = "#0284c7"     # darker blue edge

bars = ax.bar(
    x,
    monthly_eda_recent["actual_conversions"],
    color=bar_color,
    edgecolor=edge_color,
    linewidth=0.9,
    alpha=0.98,
)

# X-axis labels = trial_month
ax.set_xticks(x)
ax.set_xticklabels(
    monthly_eda_recent["trial_month"],
    rotation=30,
    ha="right"
)

ax.set_xlabel("Trial month")
ax.set_ylabel("Number of conversions")
ax.set_title("Actual conversions per month (EDA – last 12 months)", pad=18)

# Value labels on top of each bar
ax.bar_label(bars, fmt="%.0f", padding=3, fontsize=9)

# Mean & median box (small KPI box in the top-right corner)
mean_val = monthly_eda_recent["actual_conversions"].mean()
median_val = monthly_eda_recent["actual_conversions"].median()

stats_text = f"Mean: {mean_val:.1f}\nMedian: {median_val:.1f}"
ax.text(
    0.98, 0.95, stats_text,
    transform=ax.transAxes,
    ha="right",
    va="top",
    fontsize=10,
    bbox=dict(boxstyle="round,pad=0.4",
              facecolor="white",
              edgecolor="0.8")
)

# Nice headroom above the tallest bar
ax.set_ylim(0, monthly_eda_recent["actual_conversions"].max() * 1.15)

plt.tight_layout()
plt.show()

# ================================================
# XGBoost (calibrated) – validation & test tables
# ================================================
from sklearn.metrics import (
    average_precision_score,
    roc_auc_score,
    brier_score_loss,
)
import numpy as np
import pandas as pd

# ------------------------------------------------
#     Monthly tables on VALIDATION and TEST
#    (raw vs calibrated XGB, like you had for LR)
# ------------------------------------------------

# Validation – raw
val_monthly_xgb_raw = monthly_table(train.loc[is_val], p_val_xgb_raw)
print(
    "VAL monthly (XGB raw):\n",
    val_monthly_xgb_raw,
    "\n MAE mean:",
    float(val_monthly_xgb_raw["mae"].mean()),
    "MAPE mean:",
    float(val_monthly_xgb_raw["mape"].mean()),
)

# Validation – calibrated
val_monthly_xgb_cal = monthly_table(train.loc[is_val], p_val_xgb_cal)
print(
    "\nVAL monthly (XGB calibrated):\n",
    val_monthly_xgb_cal,
    "\n MAE mean:",
    float(val_monthly_xgb_cal["mae"].mean()),
    "MAPE mean:",
    float(val_monthly_xgb_cal["mape"].mean()),
)

# Test – raw
test_monthly_xgb_raw = monthly_table(test, p_test_xgb_raw)
print(
    "\nTEST monthly (XGB raw):\n",
    test_monthly_xgb_raw,
    "\n MAE mean:",
    float(test_monthly_xgb_raw["mae"].mean()),
    "MAPE mean:",
    float(test_monthly_xgb_raw["mape"].mean()),
)

# Test – calibrated
test_monthly_xgb_cal = monthly_table(test, p_test_xgb_cal)
print(
    "\nTEST monthly (XGB calibrated):\n",
    test_monthly_xgb_cal,
    "\n MAE mean:",
    float(test_monthly_xgb_cal["mae"].mean()),
    "MAPE mean:",
    float(test_monthly_xgb_cal["mape"].mean()),
)

# ------------------------------------------------
#    Single metrics table for XGB (calibrated) on TEST
#    PR-AUC, ROC-AUC, Brier, Precision@5%, Recall@5%, Monthly MAE/MAPE
# ------------------------------------------------

y_test = test["label_120d"].to_numpy(dtype=int)
k = 0.05  # top-5% operating point

# Classification metrics
pr_auc_test  = average_precision_score(y_test, p_test_xgb_cal)
roc_auc_test = roc_auc_score(y_test, p_test_xgb_cal)
brier_test   = brier_score_loss(y_test, p_test_xgb_cal)

# Precision@k and Recall@k on TEST
cutoff = np.quantile(p_test_xgb_cal, 1 - k)
topk   = (p_test_xgb_cal >= cutoff).astype(int)

tp      = ((topk == 1) & (y_test == 1)).sum()
prec_k  = tp / max(topk.sum(), 1)
rec_k   = tp / max((y_test == 1).sum(), 1)

# Monthly MAE/MAPE from the calibrated TEST table
mae_mean_test   = float(test_monthly_xgb_cal["mae"].mean())
mape_mean_test  = float(test_monthly_xgb_cal["mape"].mean())

xgb_test_summary = pd.DataFrame(
    [{
        "Model": "XGBoost (calibrated)",
        "Split": "TEST",
        "PR_AUC": pr_auc_test,
        "ROC_AUC": roc_auc_test,
        "Brier": brier_test,
        "Precision@5%": prec_k,
        "Recall@5%": rec_k,
        "Monthly_MAE_mean": mae_mean_test,
        "Monthly_MAPE_mean": mape_mean_test,
    }]
)

print("\nXGB (calibrated) – TEST metrics summary:\n", xgb_test_summary)

VAL monthly (XGB raw):
   trial_month  actual   predicted  trials        mae      mape
0     2024-10     146  140.494614    2365   5.505386  0.037708
1     2024-11     126  125.318848    2512   0.681152  0.005406
2     2024-12     115   95.758942    2257  19.241058  0.167314 
 MAE mean: 8.475865681966146 MAPE mean: 0.07014254918354455

VAL monthly (XGB calibrated):
   trial_month  actual   predicted  trials       mae      mape
0     2024-10     146  152.446042    2365  6.446042  0.044151
1     2024-11     126  128.387128    2512  2.387128  0.018945
2     2024-12     115  106.167257    2257  8.832743  0.076806 
 MAE mean: 5.888637675438076 MAPE mean: 0.04663429780582073

TEST monthly (XGB raw):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  120.735725    2496  26.264275  0.178669
1     2025-02     179  134.668793    2172  44.331207  0.247660
2     2025-03     158  131.757980    2230  26.242020  0.166089 
 MAE mean: 32.27916717529297 MAPE mean: 0.19747254749927434

TEST monthly (XGB calibrated):
   trial_month  actual   predicted  trials        mae      mape
0     2025-01     147  116.735122    2496  30.264878  0.205884
1     2025-02     179  131.280200    2172  47.719800  0.266591
2     2025-03     158  129.766261    2230  28.233739  0.178695 
 MAE mean: 35.40613902474676 MAPE mean: 0.21705637889402787

XGB (calibrated) – TEST metrics summary:
                   Model Split    PR_AUC   ROC_AUC     Brier  Precision@5%  \
0  XGBoost (calibrated)  TEST  0.354969  0.889124  0.055011        0.4275   

   Recall@5%  Monthly_MAE_mean  Monthly_MAPE_mean  
0   0.353306         35.406139           0.217056

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

# Data
val_baseline_monthly = pd.DataFrame({
    "trial_month": ["2024-10", "2024-11", "2024-12"],
    "actual": [146, 126, 115],
    "predicted": [551, 517, 448]
})

# Percent over-prediction
val_baseline_monthly["percent_over"] = (
    (val_baseline_monthly["predicted"] - val_baseline_monthly["actual"])
    / val_baseline_monthly["actual"] * 100
).round(1)

# Melt for seaborn
plot_df = val_baseline_monthly.melt(
    id_vars=["trial_month", "percent_over"],
    value_vars=["actual", "predicted"],
    var_name="Type",
    value_name="Conversions"
)

# Colors
colors = {
    "actual": "#1B84E7",      # accent blue
    "predicted": "#9AA0A6"    # soft gray
}

plt.figure(figsize=(10, 6))

ax = sns.barplot(
    data=plot_df,
    x="trial_month",
    y="Conversions",
    hue="Type",
    palette=colors
)

# ---------------------------
# Add percent labels on predicted bars ONLY
# ---------------------------
for i, row in val_baseline_monthly.iterrows():
    x = i               # the bar group
    y = row["predicted"]  # predicted height
    pct = row["percent_over"]

    ax.text(
        x, y + 10,                  # small offset above the bar
        f"+{pct}%",
        ha='center', va='bottom',
        fontsize=12,
        color="#5A5A5A",            # darker gray text
        fontweight='semibold'
    )

# Title
plt.title("Actual vs Baseline Rule — Validation Months", fontsize=18, pad=20)

# Axis labels
plt.xlabel("Trial Month", fontsize=14)
plt.ylabel("Conversions", fontsize=14)

# Legend
plt.legend(title="", fontsize=12, loc="upper right")

# Cleaner theme
sns.despine()
plt.tight_layout()

plt.show()

Feature	What	Why	How
has_business_email	0/1 flag for business-like email	Mirrors legacy rule; intuitive for marketing	lowercase → regex `contains` → `int`
agent_presence	≥1 installed agent	Core part of legacy logic	`fillna(0)` → `>=1` → `int`
agents_bin	Buckets of agent counts: `0`, `1`, `2–4`, `5+`	Clear engagement tiers	`pd.cut` on `[-1,0,1,4,∞)`; missing→`0`
event_count	Count of distinct behaviors	Compact measure of trial engagement	each col: `fillna(0)` → `>0` → sum
events_per_view	`event_count / (1 + views)`	Normalizes actions by traffic	safe division to avoid zero & outliers
pricing_intent	Viewed pricing page (0/1)	Strong intent signal	bool → `int`; missing→0
email_x_agents	Interaction: business email × agent presence	Captures legacy combo & exceptions	product of the two binary columns

Trial conversion analysis — Eden Malka¶

Load Data¶

Checking Dataset Structure and Data Types¶

Feature Overview by Category¶

Data Cleaning and Missing Values Overview¶

Feature Engineering¶

Feature Engineering Summary¶

Define Feature Buckets & Guardrails¶

Time-Aware Split for Honest Future Validation¶

Baseline heuristic: business email + at least one agent¶

Build a clean preprocess + balanced Logistic baseline¶

Guardrails Check & Tiny Dry-Run (Leakage + Transformability)¶

Train the baseline logistic model and inspect PR/ROC/Calibration¶

Calibrate the logistic probabilities (isotonic)¶

Monthly business KPIs – raw vs calibrated logistic (VAL + TEST)¶

Set up a unified evaluator for model comparison¶

Compare several candidate models on the same validation window¶

XGBoost feature importance: permutation + SHAP on validation¶

What drives conversion in the XGBoost model¶

Bar chart (Permutation Importance)¶

SHAP summary plot¶

Permutation Importance – Calibrated Logistic Regression (Comparison vs. XGBoost)¶

Comparing Logistic Regression vs. XGBoost Feature Signals¶

Final evaluation for the chosen model: XGBoost¶

Why calibration slightly hurts XGB on the test months¶

XGBoost (raw & calibrated) vs baseline: metrics + monthly forecast¶

Model Performance Comparison (TEST)¶

Monthly Conversions – Actual vs Predicted (TEST)¶

How the calibrated XGBoost model performs vs. the current rule¶

How good is the model at the Top-5% point?¶

XGB calibrated vs Baseline¶

	train	test
rows	66452	6898
cols	35	35
date_min	2023-01-01 00:00:00	2025-01-01 00:00:00
date_max	2024-12-31 00:00:00	2025-03-31 00:00:00

	CHANNEL_ID	CAMPAIGN_ID	DATE	TRIAL_CONVERTED_TO_PURCHASE_DATE	IS_TRIAL_CONVERTED_TO_PURCHASE	NUM_OF_INSTALLED_AGENTS	IP_COUNTRY	REGISTRATION_EMAIL_TYPE	COMPANY_TYPE	INITIAL_ATERA_GOAL	...	'technician added'	CLEARBIT_SECTOR	CLEARBIT_INDUSTRY_GROUP	CLEARBIT_INDUSTRY	VIEWED_PRICING_PAGE	HUBSPOT_CLICKED	SEGMENT_EVENTS	DISTINCT_SEGMENT_EVENTS	SEGMENT_PAGE_VIEWS	DISTINCT_SEGMENT_PAGE_VIEWS
0	37	406	2023-08-27	NaT	False	0.0	France	business	Other	None	...	NaN	None	None	None	False	0.0	9.0	9.0	NaN	NaN
1	2	218	2023-12-03	NaT	False	1.0	Romania	business	MSP	None	...	NaN	None	None	None	True	0.0	254.0	106.0	154.0	58.0

	CHANNEL_ID	CAMPAIGN_ID	DATE	TRIAL_CONVERTED_TO_PURCHASE_DATE	IS_TRIAL_CONVERTED_TO_PURCHASE	NUM_OF_INSTALLED_AGENTS	IP_COUNTRY	REGISTRATION_EMAIL_TYPE	COMPANY_TYPE	INITIAL_ATERA_GOAL	...	'technician added'	CLEARBIT_SECTOR	CLEARBIT_INDUSTRY_GROUP	CLEARBIT_INDUSTRY	VIEWED_PRICING_PAGE	HUBSPOT_CLICKED	SEGMENT_EVENTS	DISTINCT_SEGMENT_EVENTS	SEGMENT_PAGE_VIEWS	DISTINCT_SEGMENT_PAGE_VIEWS
0	19	204	2025-01-13	NaT	False	0.0	India	free	IT Department	None	...	NaN	None	None	None	False	0.0	36.0	30.0	25.0	12.0
1	52	218	2025-03-21	NaT	False	0.0	Dominican Republic	business	IT Department	None	...	NaN	None	None	None	False	0.0	14.0	14.0	NaN	NaN

	Model	PR_AUC	ROC_AUC	Brier	Precision@5%	Recall@5%	Monthly_MAE_mean	Monthly_MAPE_mean
0	XGBoost	0.319	0.903	0.043	0.356	0.328	8.476	0.070
1	LogReg_balanced (calibrated)	0.315	0.901	0.042	0.297	0.517	9.180	0.074
2	RandomForest	0.283	0.884	0.044	0.300	0.276	7.778	0.062