224 lines
7.7 KiB
Python
224 lines
7.7 KiB
Python
|
import threading # 放在你的其他 import 之前
|
|||
|
|
if not hasattr(threading.Thread, "isAlive"):
|
|||
|
|
threading.Thread.isAlive = threading.Thread.is_alive # 给旧调试器一个别名
|
|||
|
|
|
|||
|
|
import warnings
|
|||
|
|
import os
|
|||
|
|
import numpy as np
|
|||
|
|
import pandas as pd
|
|||
|
|
from scipy import stats
|
|||
|
|
|
|||
|
|
warnings.filterwarnings("ignore")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def detect_outliers_iqr(data: pd.DataFrame, column: str) -> pd.Series:
|
|||
|
|
"""使用 IQR 方法检测异常值，返回与 data 同索引的布尔序列"""
|
|||
|
|
s = pd.to_numeric(data[column], errors="coerce")
|
|||
|
|
q1 = s.quantile(0.25)
|
|||
|
|
q3 = s.quantile(0.75)
|
|||
|
|
iqr = q3 - q1
|
|||
|
|
lower = q1 - 1.5 * iqr
|
|||
|
|
upper = q3 + 1.5 * iqr
|
|||
|
|
mask = (s < lower) | (s > upper)
|
|||
|
|
# 对 NaN 不判为异常
|
|||
|
|
mask = mask.fillna(False)
|
|||
|
|
mask.index = data.index
|
|||
|
|
return mask
|
|||
|
|
|
|||
|
|
|
|||
|
|
def detect_outliers_zscore(data: pd.DataFrame, column: str, threshold: float = 3.0) -> pd.Series:
|
|||
|
|
"""使用 Z-score 方法检测异常值，返回与 data 同索引的布尔序列"""
|
|||
|
|
s = pd.to_numeric(data[column], errors="coerce")
|
|||
|
|
z = pd.Series(stats.zscore(s.dropna()), index=s.dropna().index)
|
|||
|
|
mask = (z.abs() > threshold).reindex(data.index).fillna(False)
|
|||
|
|
return mask
|
|||
|
|
|
|||
|
|
|
|||
|
|
def read_csv_robust(path, **kwargs):
|
|||
|
|
"""
|
|||
|
|
尝试多种编码读取 CSV；成功即返回 DataFrame。
|
|||
|
|
kwargs 会透传给 pd.read_csv（比如 sep、dtype 等）。
|
|||
|
|
"""
|
|||
|
|
# 按出现概率排序；优先无损 + 常见中文编码
|
|||
|
|
encodings = [
|
|||
|
|
"utf-8", "utf-8-sig",
|
|||
|
|
"gbk", "gb18030", "cp936",
|
|||
|
|
"utf-16", "utf-16le", "utf-16be",
|
|||
|
|
"cp1252", "big5",
|
|||
|
|
"latin1", # 最后兜底（能读但中文会变乱码）
|
|||
|
|
]
|
|||
|
|
errors_modes = ["strict", "replace"] # 先严格，失败再替换非法字符
|
|||
|
|
|
|||
|
|
last_err = None
|
|||
|
|
for enc in encodings:
|
|||
|
|
for emode in errors_modes:
|
|||
|
|
try:
|
|||
|
|
return pd.read_csv(path, encoding=enc, **kwargs)
|
|||
|
|
except Exception as e:
|
|||
|
|
last_err = e
|
|||
|
|
continue
|
|||
|
|
# 如果全失败，抛出最后一个错误
|
|||
|
|
raise last_err
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _decimal_len(v) -> float:
|
|||
|
|
"""计算数值或字符串小数点后的位数；若无法计算返回 NaN"""
|
|||
|
|
if pd.isna(v):
|
|||
|
|
return np.nan
|
|||
|
|
try:
|
|||
|
|
# 统一成字符串处理
|
|||
|
|
s = str(v)
|
|||
|
|
if "." not in s:
|
|||
|
|
return 0
|
|||
|
|
frac = s.split(".", 1)[1]
|
|||
|
|
# 去掉科学计数法中的尾随部分（如 '1.234e-05'）
|
|||
|
|
frac = frac.split("e")[0].split("E")[0]
|
|||
|
|
return len(frac)
|
|||
|
|
except Exception:
|
|||
|
|
return np.nan
|
|||
|
|
|
|||
|
|
|
|||
|
|
def process_water_quality_data(input_file: str, output_file: str):
|
|||
|
|
"""
|
|||
|
|
处理水质数据 CSV 文件
|
|||
|
|
|
|||
|
|
参数:
|
|||
|
|
input_file: 输入 CSV 文件路径
|
|||
|
|
output_file: 输出 CSV 文件路径
|
|||
|
|
"""
|
|||
|
|
# 0) 读取
|
|||
|
|
print("正在读取 CSV 文件...")
|
|||
|
|
df = read_csv_robust(input_file)
|
|||
|
|
print(f"原始数据形状: {df.shape}")
|
|||
|
|
print(f"列名: {list(df.columns)}")
|
|||
|
|
|
|||
|
|
# 1) 经纬度精度筛选（小数位 >= 7）
|
|||
|
|
print("\n正在筛选经纬度精度(小数位>=7)...")
|
|||
|
|
initial_count = len(df)
|
|||
|
|
|
|||
|
|
for col in ["经度", "纬度"]:
|
|||
|
|
if col in df.columns:
|
|||
|
|
dec_len = df[col].apply(_decimal_len)
|
|||
|
|
keep_mask = dec_len >= 7
|
|||
|
|
dropped = (~keep_mask).sum()
|
|||
|
|
df = df[keep_mask].copy()
|
|||
|
|
print(f"列 {col}: 去除了 {int(dropped)} 行（保留 {len(df)} 行）")
|
|||
|
|
|
|||
|
|
after_coord_filter = len(df)
|
|||
|
|
print(f"经纬度精度筛选后剩余: {after_coord_filter} 行 (去除了 {initial_count - after_coord_filter} 行)")
|
|||
|
|
|
|||
|
|
# 2) 异常值检测（IQR）- 只删除异常值，不删除整行
|
|||
|
|
print("\n正在检测异常值(IQR)...")
|
|||
|
|
# 数值列
|
|||
|
|
numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
|
|||
|
|
|
|||
|
|
# 排除不检测的列
|
|||
|
|
exclude_columns = ["时间", "测量点", "纬度", "经度"]
|
|||
|
|
if "原始" in df.columns:
|
|||
|
|
exclude_columns.append("原始")
|
|||
|
|
|
|||
|
|
columns_to_check = [c for c in numeric_columns if c not in exclude_columns]
|
|||
|
|
print(f"将检测以下列的异常值: {columns_to_check}")
|
|||
|
|
|
|||
|
|
df_clean = df.copy()
|
|||
|
|
total_outliers_removed = 0
|
|||
|
|
|
|||
|
|
for column in columns_to_check:
|
|||
|
|
if column in df_clean.columns and df_clean[column].notna().sum() > 0:
|
|||
|
|
col_mask = detect_outliers_iqr(df_clean, column)
|
|||
|
|
outlier_count = int(col_mask.sum())
|
|||
|
|
print(f'列 "{column}" 检测到 {outlier_count} 个异常值，将其设为 NaN')
|
|||
|
|
# 只将异常值设为 NaN，不删除整行
|
|||
|
|
df_clean.loc[col_mask, column] = np.nan
|
|||
|
|
total_outliers_removed += outlier_count
|
|||
|
|
|
|||
|
|
after_outlier_filter = len(df_clean)
|
|||
|
|
print(f"异常值处理完成: 保留 {after_outlier_filter} 行数据，共处理了 {total_outliers_removed} 个异常值")
|
|||
|
|
|
|||
|
|
# 3) 去除 "原始" 列（若存在）
|
|||
|
|
if "原始" in df_clean.columns:
|
|||
|
|
df_clean = df_clean.drop(columns=["原始"])
|
|||
|
|
print('已去除 "原始" 列')
|
|||
|
|
|
|||
|
|
# 4) 字段类型处理：尽量把“时间”转为 datetime
|
|||
|
|
if "时间" in df_clean.columns:
|
|||
|
|
try:
|
|||
|
|
df_clean["时间"] = pd.to_datetime(df_clean["时间"], errors="coerce")
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
# 5) 按测量点统计平均值
|
|||
|
|
print("\n正在按测量点统计平均值...")
|
|||
|
|
if "测量点" not in df_clean.columns:
|
|||
|
|
print('错误：未找到 "测量点" 列')
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
# 构建聚合字典
|
|||
|
|
agg_dict = {}
|
|||
|
|
if "时间" in df_clean.columns and np.issubdtype(df_clean["时间"].dtype, np.datetime64):
|
|||
|
|
# 时间取平均（等价于时间戳平均）
|
|||
|
|
agg_dict["时间"] = "mean"
|
|||
|
|
elif "时间" in df_clean.columns:
|
|||
|
|
# 如果不是时间类型，保留最常见值以避免无意义的字符串平均
|
|||
|
|
agg_dict["时间"] = lambda s: s.mode().iloc[0] if not s.mode().empty else s.dropna().iloc[0] if s.dropna().size else np.nan
|
|||
|
|
|
|||
|
|
for col in ["纬度", "经度"]:
|
|||
|
|
if col in df_clean.columns:
|
|||
|
|
agg_dict[col] = "mean"
|
|||
|
|
|
|||
|
|
# 其余数值列取均值
|
|||
|
|
for col in df_clean.select_dtypes(include=[np.number]).columns:
|
|||
|
|
if col not in ["纬度", "经度"]:
|
|||
|
|
agg_dict[col] = "mean"
|
|||
|
|
|
|||
|
|
grouped = df_clean.groupby("测量点", as_index=False).agg(agg_dict)
|
|||
|
|
|
|||
|
|
print(f"统计完成，共 {len(grouped)} 个测量点")
|
|||
|
|
print(f"输出数据形状: {grouped.shape}")
|
|||
|
|
|
|||
|
|
# 6) 去除"时间"和"测量点"列
|
|||
|
|
columns_to_drop = []
|
|||
|
|
if "时间" in grouped.columns:
|
|||
|
|
columns_to_drop.append("时间")
|
|||
|
|
if "测量点" in grouped.columns:
|
|||
|
|
columns_to_drop.append("测量点")
|
|||
|
|
|
|||
|
|
if columns_to_drop:
|
|||
|
|
grouped = grouped.drop(columns=columns_to_drop)
|
|||
|
|
print(f"已去除列: {columns_to_drop}")
|
|||
|
|
print(f"去除列后数据形状: {grouped.shape}")
|
|||
|
|
|
|||
|
|
# 7) 保存
|
|||
|
|
os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
|
|||
|
|
grouped.to_csv(output_file, index=False, encoding="utf-8-sig")
|
|||
|
|
print(f"\n处理完成！结果已保存到: {output_file}")
|
|||
|
|
|
|||
|
|
# 摘要
|
|||
|
|
print("\n=== 处理结果摘要 ===")
|
|||
|
|
print(f"原始数据行数: {initial_count}")
|
|||
|
|
print(f"经纬度精度筛选后: {after_coord_filter}")
|
|||
|
|
print(f"异常值筛选后: {after_outlier_filter}")
|
|||
|
|
print(f"最终统计结果: {len(grouped)} 个测量点")
|
|||
|
|
|
|||
|
|
return grouped
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
"""主函数"""
|
|||
|
|
input_file = r"D:\BaiduNetdiskDownload\yaobao\csv\input.csv"
|
|||
|
|
output_file =r"D:\BaiduNetdiskDownload\yaobao\csv\output_test.csv"
|
|||
|
|
|
|||
|
|
if not output_file:
|
|||
|
|
output_file = "processed_water_quality.csv"
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
_ = process_water_quality_data(input_file, output_file)
|
|||
|
|
except FileNotFoundError as e:
|
|||
|
|
print(f"文件未找到：{e}")
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"处理失败：{e}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|