AI训练师三级 - 实操知识卡片

# 类型转换
data['Age'] = data['Age'].astype(int)
data['Amount'] = data['Amount'].astype(float)

# 安全转换（处理非数值）
data['hp'] = pd.to_numeric(data['hp'], errors='coerce')
data = data.dropna()

# 范围过滤 — 常见阈值
data = data[
    (data['Age'].between(18, 70)) &
    (data['Amount'] > 0) &
    (data['Score'].between(1, 5))
]

# 单条件标记（1.1.1 风险等级）
data['RiskLevel'] = np.where(
    data['DaysInHospital'] > 7,
    '高风险', '低风险'
)

# 多条件异常标记（1.1.2 温度异常）
data['IsAbnormal'] = np.where(
    (data['Temperature'] < -10) | (data['Temperature'] > 50),
    '异常', '正常'
)

# 统计标记结果
print(data['RiskLevel'].value_counts())

# 计算比例
high = (data['RiskLevel'] == '高风险').sum()
total = len(data)
ratio = high / total

# BMI分箱（1.1.1）
bins = [0, 18.5, 24, 28, float('inf')]
labels = ['偏瘦', '正常', '超重', '肥胖']
data['BMIRange'] = pd.cut(
    data['BMI'], bins=bins, labels=labels
)

# 年龄分箱（1.1.1 / 1.1.5）
age_bins = [0, 26, 36, 46, 56, 66, float('inf')]
age_labels = ['≤25岁', '26-35岁', '36-45岁',
              '46-55岁', '56-65岁', '＞65岁']
data['AgeRange'] = pd.cut(
    data['Age'], bins=age_bins, labels=age_labels
)

# 频次统计
print(data['Category'].value_counts())

# 单列分组均值
result = data.groupby('Gender')['Amount'].mean()

# 多列分组聚合
result = data.groupby('Gender').agg({
    'Speed': 'mean',
    'Distance': 'mean',
    'Time': 'mean'
})

# 分箱后统计（count + mean）
result = data.groupby('SensorType').agg({
    'Value': ['count', 'mean']
})

# 分箱后统计风险比例
risk = data.groupby('BMIRange')['RiskLevel'].apply(
    lambda x: (x == '高风险').sum() / len(x)
)

# Z-score 标准化
mean_val = data['Amount'].mean()
std_val = data['Amount'].std()
data['Amount_std'] = (data['Amount'] - mean_val) / std_val

# 缺失值填充（前向 + 后向）
data['Value'] = data['Value'].fillna(method='ffill')
data['Value'] = data['Value'].fillna(method='bfill')

# 保存结果
data.to_csv('cleaned_data.csv', index=False)

from sklearn.preprocessing import StandardScaler, MinMaxScaler

# StandardScaler: 均值0, 标准差1（2.1.1, 2.1.2）
scaler = StandardScaler()
numerical = ['displacement', 'horsepower', 'weight']
data[numerical] = scaler.fit_transform(data[numerical])

# MinMaxScaler: 缩放到 [0, 1]（2.1.3）
scaler = MinMaxScaler()
data[numerical] = scaler.fit_transform(data[numerical])

# 四分位距 (IQR) 法去除异常值
Q1 = data.quantile(0.25)
Q3 = data.quantile(0.75)
IQR = Q3 - Q1

# 删除超出 1.5倍IQR 的数据
data = data[~(
    (data < (Q1 - 1.5 * IQR)) |
    (data > (Q3 + 1.5 * IQR))
).any(axis=1)]

from sklearn.model_selection import train_test_split

# 方式一：指定特征列
feature_columns = ['feat1', 'feat2', 'feat3']
X = data[feature_columns]
y = data['target']

# 方式二：排除目标列
X = data.drop(columns=['target'])
y = data['target']

# 80/20 划分 — 固定参数！
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 保存
cleaned = pd.concat([X, y], axis=1)
cleaned.to_csv('cleaned_data.csv', index=False)

import pickle
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report
from imblearn.over_sampling import SMOTE

# 训练逻辑回归
model = LogisticRegression(max_iter=1000)
model.fit(X_train, y_train)

# 保存模型
with open('model.pkl', 'wb') as f:
    pickle.dump(model, f)

# 预测 & 评估
y_pred = model.predict(X_test)
accuracy = model.score(X_test, y_test)
print(classification_report(y_test, y_pred))

# ===== SMOTE 纠错 =====
smote = SMOTE(random_state=42)
X_res, y_res = smote.fit_resample(X_train, y_train)
model.fit(X_res, y_res)
y_pred_new = model.predict(X_test)
accuracy_new = model.score(X_test, y_test)

from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, r2_score
from xgboost import XGBRegressor

# ---- 线性回归 (Pipeline) ----
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('linreg', LinearRegression())
])
pipeline.fit(X_train, y_train)
y_pred = pipeline.predict(X_test)

# ---- 随机森林 ----
rf = RandomForestRegressor(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
y_pred_rf = rf.predict(X_test)

# ---- 决策树 ----
dt = DecisionTreeRegressor(random_state=42)
dt.fit(X_train, y_train)

# ---- XGBoost 纠错 ----
xgb = XGBRegressor(n_estimators=100, random_state=42)
xgb.fit(X_train, y_train)
y_pred_xgb = xgb.predict(X_test)

# 训练集得分
train_score = model.score(X_train, y_train)
# 测试集得分
test_score = model.score(X_test, y_test)
# 均方误差
mse = mean_squared_error(y_test, y_pred)
# R² 决定系数
r2 = r2_score(y_test, y_pred)

# 保存模型
with open('model.pkl', 'wb') as f:
    pickle.dump(model, f)

# 保存预测结果
results = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
results.to_csv('results.txt', index=False)

import numpy as np
import onnxruntime as ort
from PIL import Image
from scipy.special import softmax

# 1. 加载模型
session = ort.InferenceSession('onnx/resnet.onnx')
input_name = session.get_inputs()[0].name
output_name = session.get_outputs()[0].name

# 2. 加载标签
with open('onnx/labels.txt', 'r') as f:
    labels = [line.strip() for line in f.readlines()]

# 3. 图像预处理
def preprocess_image(path, size=224):
    img = Image.open(path).convert('RGB')
    img = img.resize((size, size))
    arr = np.array(img).astype(np.float32) / 255.0
    # ImageNet 标准化
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    arr = (arr - mean) / std
    # HWC → CHW → NCHW
    arr = arr.transpose(2, 0, 1)
    arr = np.expand_dims(arr, axis=0).astype(np.float32)
    return arr

# 4. 推理
processed = preprocess_image('img_test.jpg')
output = session.run(
    [output_name], {input_name: processed}
)[0]

# 5. Top-5 结果
probs = softmax(output, axis=-1)
top5 = np.argsort(probs[0])[-5:][::-1]
for i, idx in enumerate(top5):
    print(f"  {i+1}. {labels[idx]}: {probs[0][idx]:.4f}")

# MNIST: 灰度图, 28x28, 不需要 ImageNet 标准化
def preprocess_mnist(path):
    img = Image.open(path).convert('L')      # 灰度！
    img = img.resize((28, 28))              # 28x28！
    arr = np.array(img).astype(np.float32) / 255.0
    arr = np.expand_dims(arr, axis=(0, 1)) # (1,1,28,28)
    return arr

processed = preprocess_mnist('img_test.png')
output = session.run([output_name], {input_name: processed})[0]
digit = np.argmax(output)
print(f"识别结果: {digit}")