别再用Excel做AI数据分析了，真正高手从不这么干

# 数据处理与分析
import pandas as pd  # 强大的数据处理工具
import numpy as np   # 科学计算基础库

# 数据可视化
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px

# 机器学习与深度学习
import scikit-learn as sklearn  # 传统机器学习算法
import tensorflow as tf        # Google开发的深度学习框架
import torch                   # Facebook开发的动态图深度学习框架

# 自然语言处理
import nltk
import spacy
import transformers

# 数据处理
library(dplyr)    # 数据操作
library(tidyr)    # 数据整理
library(data.table) # 大数据高效处理

# 数据可视化
library(ggplot2)  # 优雅的数据可视化
library(plotly)   # 交互式图表
library(shiny)    # 交互式Web应用

# 机器学习
library(caret)    # 机器学习训练框架
library(randomForest) # 随机森林算法
library(xgboost)     # 梯度提升树

# 使用API获取数据
import requests

response = requests.get("https://turingpub.com/data")
data = response.json()

# 使用pandas进行数据清洗
import pandas as pd

df = pd.DataFrame(data)
# 处理缺失值
df = df.dropna()
# 转换数据类型
df['date'] = pd.to_datetime(df['date'])
# 去除重复项
df = df.drop_duplicates()

# 基本统计信息
print(df.describe())

# 数据可视化
import matplotlib.pyplot as plt
import seaborn as sns

# 分布图
sns.histplot(df['value'])
plt.show()

# 相关性热图
correlation = df.corr()
sns.heatmap(correlation, annot=True)
plt.show()

# 创建新特征
df['new_feature'] = df['feature1'] / df['feature2']

# 特征转换
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
df['scaled_feature'] = scaler.fit_transform(df[['feature']])

# 特征选择
from sklearn.feature_selection import SelectKBest, f_classif

selector = SelectKBest(f_classif, k=10)
X_new = selector.fit_transform(X, y)

from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# 创建模型
model = RandomForestClassifier()

# 交叉验证
scores = cross_val_score(model, X, y, cv=5)
print(f"平均准确率: {scores.mean():.2f}")

# 训练模型
model.fit(X_train, y_train)

# 预测和评估
y_pred = model.predict(X_test)
print(f"准确率: {accuracy_score(y_test, y_pred)}")
print(f"精确率: {precision_score(y_test, y_pred)}")
print(f"召回率: {recall_score(y_test, y_pred)}")
print(f"F1分数: {f1_score(y_test, y_pred)}")

# 特征重要性
importances = model.feature_importances_
features = X.columns
indices = np.argsort(importances)[-10:]  # 最重要的10个特征

plt.figure(figsize=(12, 8))
plt.title('特征重要性')
plt.barh(range(len(indices)), importances[indices], color='b', align='center')
plt.yticks(range(len(indices)), [features[i] for i in indices])
plt.xlabel('相对重要性')
plt.show()

# 使用SHAP进行模型解释
import shap

explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)

# 可视化第一个预测的解释
shap.force_plot(explainer.expected_value[1], shap_values[1][0,:], X_test.iloc[0,:])