数据抓取失败：寻求帮助

Question

我尝试了以下方法：

• 利用德国证券交易所的 API (https://api.boerse-frankfurt.de/v1/search/equity_search) 检索指数值。
• 可以使用参数从外部访问API。通常会从 https://www.boerse-frankfurt.de/index/dax/zugehoerige-werte 调用。
• 这里有一个类似的问题（Extract or generated X-Client-TraceId for header in GET-request），但我正在尝试查询不同的API端点。
• 这里有一篇博客文章（https://lwthiker.com/reversing/2022/02/12/analyzing-stock-exchange-api.html），但我无法适应我的场景。
• 我怀疑我的网址组成不正确。

这是我的源代码：

import datetime
import hashlib
import json
import requests
from pprint import pprint

def generate_headers(url):
    salt = "w4ivc1ATTGta6njAZzMbkL3kJwxMfEAKDa3MNr"
    current_time = datetime.datetime.now(tz=datetime.timezone.utc)
    client_date = (current_time
        .isoformat(timespec="milliseconds")
        .replace("+00:00", "Z")
    )
    client_traceid = hashlib.md5(
        (client_date + url + salt).encode("utf-8")
    )
    security = hashlib.md5(
        current_time.strftime("%Y%m%d%H%M").encode("utf-8")
    )

    return {
        "Client-Date": client_date,
        "X-Client-TraceId": client_traceid.hexdigest(),
        "X-Security": security.hexdigest()
    }

payload = {"indices"  : ["DE0007203275"],
           "lang"     : "de",
           "offset"   : 0,
           "limit"    : 50,
           "sorting"  : "TURNOVER",
           "sortOrder": "DESC"}

urlString = "https://api.boerse-frankfurt.de/v1/search/equity_search?indices=['DE0007203275']&lang=de&offset=0&limit=50&sorting=TURNOVER&sortOrder=DESC"

result = generate_headers(urlString)
print(result)

headers = {"Host"             : "api.boerse-frankfurt.de",
           "Referer"          : "https://www.boerse-frankfurt.de/",
           "Content-Type"     : "application/json; charset=utf-8",
           "Client-Date"      : result["Client-Date"],     
           "X-Client-TraceId" : result["X-Client-TraceId"],  
           "X-Security"       : result["X-Security"]
           }

response = requests.post(url,
                         headers=headers,
                         json=payload)

pprint(json.loads(response.content))

欢迎任何建议。谢谢！！！

Answer 1

尝试：

import hashlib
from datetime import datetime

import requests


def get_trace_id():
    timeutc = datetime.utcnow()
    timestr = timeutc.isoformat(timespec="milliseconds") + "Z"

    traceidbase = timestr + url + "w4ivc1ATTGta6njAZzMbkL3kJwxMfEAKDa3MNr"
    encoded = traceidbase.encode()
    traceid = hashlib.md5(encoded).hexdigest()

    return timestr, traceid


url = "https://api.boerse-frankfurt.de/v1/search/equity_search"

payload = {
    "indices": ["DE0008469008"],
    "lang": "de",
    "limit": 25,
    "offset": 0,
    "sorting": "TURNOVER",
    "sortOrder": "DESC",
}

client_date, trace_id = get_trace_id()

headers = {
    "X-Client-TraceId": trace_id,
    "Client-Date": client_date,
}

data = requests.post(url, headers=headers, json=payload).json()

print(json.dumps(data, indent=4))

打印：

{
    "recordsTotal": 40,
    "data": [
        {
            "turnover": 353383054.2,
            "esgRiskScore": null,
            "industryAverage": null,
            "changeToPrevDay": 0.1915708812,
            "weeks52High": 571.8,
            "weeks52Low": 226.5,
            "performance3Month": 71.41920682,
            "performance1Year": 101.69687621,
            "performance3Year": 496.21523028,
            "tradingUnitsToday": null,
            "marketCapitalisation": 22781880000.0,
            "name": {
                "originalValue": "RHEINMETALL AG",
                "translations": {
                    "others": "Rheinmetall AG"
                }
            },
            "wkn": "703000",
            "isin": "DE0007030009",
            "slug": "rheinmetall-ag",
            "overview": {
                "exchange": "XETR",
                "lastPrice": 523.0,
                "changeToPrevDay": 0.1915708812,
                "dateTimeLastPrice": "2024-04-10T17:37:03+02:00",
                "turnover": 353383054.2,
                "tradingUnitsToday": null
            },
            "performance": {
                "weeks52High": 571.8,
                "weeks52Low": 226.5,
                "performance3Month": 71.41920682,
                "performance6Month": 108.78243513,
                "performance1Year": 101.69687621,
                "performance3Year": 496.21523028
            },
            "keyData": {
                "priceEarningsRatio": 23.2873,
                "dividendPerShare": 5.7,
                "dividendPerShareExtra": null,
                "dividendRatio": 1.092,
                "earningsPerShareGross": 17.76,
                "earningsPerShareBasic": 12.32,
                "priceBookRatio": 3.7567,
                "marketCapitalisation": 22781880000.0
            },
            "sustainability": {
                "esgRiskScore": null,
                "industryAverage": null
            }
        },

...