ZDXX/2_1banben/services/crawler_82.py

# services/crawler_82.py
import requests
import json
import logging
from lxml import etree
from config import Config
from datetime import datetime

CONFIG = Config.CRAWLER_CONFIG["82"]


def run_82_logic():
    """返回 result_list"""
    results = []
    print(">>> [82爬虫] 启动...")
    session = requests.Session()

    try:
        session.post(f"{CONFIG['base_url']}/login.php", data=CONFIG["login"], timeout=10)
        resp = session.post(f"{CONFIG['base_url']}/GetStationList.php", timeout=10)
        html = etree.HTML(resp.content)
        if html is None: return []

        stations = html.xpath('//option/@value')
        for sid in [s for s in stations if s]:
            data_packet = {
                'source': '82网站',
                'name': str(sid),
                'status': '正常',
                'value': '',
                # 同 106：默认留空，只有真正拿到数据时间才覆盖，
                # 不用 datetime.now() 冒充数据时间
                'target_time': None,
                'raw_json': {},
                'temp_file': None
            }
            try:
                r = session.post(f"{CONFIG['base_url']}/getLastWeatherData.php", data=str(sid),
                                 headers={'Content-Type': 'text/plain'}, timeout=10)
                try:
                    data = r.json()
                except:
                    data = None

                if data:
                    d_list = data.get('date', [])
                    # 拿不到时间时必须是 None，不能是 "N/A" —— 字符串在入库层是真值，
                    # 会把设备主表已经冻结的有效 latest_time 覆盖掉，令 offset 倒退成"从未同步"
                    latest = str(d_list[-1]) if d_list else None
                    data_packet['target_time'] = latest
                    data_packet['value'] = f"Data Points: {len(d_list)}"
                    data_packet['raw_json'] = data  # 🔥 存完整JSON
                else:
                    data_packet['status'] = '异常'
                    data_packet['value'] = "返回空数据"

            except Exception as e:
                data_packet['status'] = '异常'
                data_packet['value'] = "单个采集失败"

            results.append(data_packet)

    except Exception as e:
        logging.error(f"82 Crawler Error: {e}")

    return results