HttpWebRequest 突然没有响应

Question

我在公共网站上运行 HttpWebRequest 以返回 Json 至少有几年了。上周它突然停止工作，并且不会通过此代码给出响应（它被称为“旧”，因为我正在使用带有 powershell 脚本的解决方法，但这是一个笨拙的解决方案）。它最终会在默认的 100 秒后超时，但更长的超时时间无法修复它。

    Function CreateRequestOld(ByVal sMESUser As String, ByVal sMeeting As String, ByVal sRaceDate As String, ByVal iRaceNumber As Integer, ByVal iTrade As Integer, Optional ByVal sJuristriction As String = "VIC") As String
        Dim Url As String
        Dim request As HttpWebRequest = Nothing
        Dim dataStream As Stream = Nothing
        Dim response As HttpWebResponse = Nothing
        Dim strResponseStatus As String = ""
        Dim reader As StreamReader = Nothing
        Dim responseFromServer As String = ""
        Dim sFile As String = "C:\logs\MESRaceLog_" & sMESUser & ".txt"
        Dim Util As New Utilities
        Dim sMeetingCode As String
        Dim RI As New RaceInfo
        Dim dte As New DateClass
        Dim dtRaceDate As Date

        If (iTrade = 1) Then
            sFile = "C:\logs\MESRaceTradeLog_" & sMESUser & ".txt"
        End If
        If (iTrade = 2) Then
            sFile = "C:\logs\MESRaceOddsLog_" & sMESUser & ".txt"
        End If

        Try
            sMeetingCode = RI.GetTABCouseCode(sMeeting)
            If (sMeetingCode = "") Then
                Util.LogMsg(sFile, "TAB CreateRequest Error no meeting code for Meeting name " & sMeeting)
                Return ""
            End If

            dtRaceDate = dte.ConvertStringToDate(sRaceDate, "dd/mm/yyyy")
            sRaceDate = dte.Get_Date_String(dtRaceDate, "yyyy-mm-dd")

            Url = "https://api.beta.tab.com.au/v1/tab-info-service/racing/dates/" & sRaceDate & "/meetings/R/" & sMeetingCode & "/races/" & iRaceNumber & "?jurisdiction=" & sJuristriction
            Util.LogMsg(sFile, "TAB CreateRequest URL " & Url)

            request = HttpWebRequest.Create(New Uri(Url))
            request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
            request.Method = "GET"
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
            request.AllowAutoRedirect = True
            response = request.GetResponse()

            strResponseStatus = CType(response, HttpWebResponse).StatusDescription
            dataStream = response.GetResponseStream()
            reader = New StreamReader(dataStream)
            responseFromServer = reader.ReadToEnd()
            reader.Close()
            dataStream.Close()
            response.Close()

        Catch ex As Exception
            Util.LogMsg(sFile, "CreateRequest Error" & vbCrLf & ex.Message)
        End Try
        Return responseFromServer
    End Function

但是，如果我将 URL 粘贴到浏览器（在我的例子中是 Chrome）中，它会按预期返回 Json。

所以我尝试了 Postman，如果只是简单的 URL，它也不会响应。但是，如果我复制curl脚本并使用Postman，它确实会正确响应。我注意到，curl 脚本有 23 个请求标头元素，而普通 URL（失败的那个）只有 5 个标头元素。

我怀疑该网站已更改为 HTTP/2，因此需要所有这些额外的标头元素。因此，我从 Chrome 中将 URL 复制为 Windows Powershell（在开发人员工具下，右键单击，复制为 Windows Powershee），并创建了一个如下脚本，该脚本运行良好并获取响应。

$session = New-Object Microsoft.PowerShell.Commands.WebRequestSession
$session.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
$session.Cookies.Add((New-Object System.Net.Cookie("_cs_c", "0", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_tgpc", "d077be37-9bfe-58ce-bb34-2219fe18534c", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("s_ecid", "MCMID%7C42816005962630671342879997247751714175", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_gcl_au", "1.1.1461016396.1701314243", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("__qca", "P0-115178265-1701314243095", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_fbp", "fb.2.1701314243684.710804252", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_scid", "173c7c5f-d2aa-4407-9b6d-5eb48d7d5326", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_scid_r", "173c7c5f-d2aa-4407-9b6d-5eb48d7d5326", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_sctr", "1%7C1706792400000", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("at_check", "true", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("AMCVS_5C750C3A53DB4D070A490D4D%40AdobeOrg", "1", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_abck", "E98C9DC841935F14412438E52AEE528E~0~YAAQBcfOF59EHWuNAQAAsVHf0gvjwuRSy0ElwTKYqvAoJLQPTUBCXrzWbnUSm592eN9RKleGSHb6N1G4sOpkqms4YYyspcf+3j26XCyZ5cUO/PtysyIwLocjO/mDFZ0aQTYZIcMXSuuNycWI0uarkBykoIE5TBfeo6kPD1jD7LecoSHOPVRT/z/FC/0XUS8QcgfqHESSh7Gk6D8Dh3OBQ67qpu6GXq2enMxvn/8v0vn9u0DfH739OwrQyIfO82L11CRPswZtzXj0Wk5Qx5z9ObdAZ+eM9Edz+ts4SjQVXm585PwP9aam/xPL6Lqhov+8+htIFbaKT9O9WoemaBfthkvFk90cFfZ4tv57iPDPwT7+dIDGIcwph6wK5gsd9DS4kCpPXiOIgDcJIV4r6/tL2h8xgm+Ui+Sx~-1~-1~-1", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("bm_sz", "55EEBE064EF9FD3569F82C5AE9083857~YAAQBcfOF6FEHWuNAQAAsVHf0had2HtUmx7KB2ZnSoNh2rj59HNbyA1TAzJCjLmz8aXSPpV1Q699Pfux/9L01gXAmWItcxkttXyzNTYJBwN5/0YXC7Z5KMVxPA4YnDI+DPEUDgbo9pls/PxE86j6qFbQsHPk5mjH5trCuBl4I6lLa4we6cHSZZ2D8Rb8hNSdi3hfkgNtXDISvW35NyOo3EMgBYkb3SZL5BgjWsI6D/Bw9rpj80MRJs/NPZPHOaCvj76NQEHEr2FNclSS1j5sKa3dCeLQC3CfFtw705FlSHX+fxLCaJr0k+V3guclD9JQxD64qzawCSqOur3H8sGNdd6eRHPULHeijC+naKplTbnYgTtwUX2KdjE=~3621429~4469301", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("AKA_A2", "A", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("ak_bmsc", "F23866DFAFE8CC0C588C07C641D011C3~000000000000000000000000000000~YAAQOU5haKnFDcqNAQAAbNln0xZzvuAFiiDMoeSqPqgw/Qyd1vMSeWW1tBvtNwt1/1xz7GqrMaHlb6o/7sgtgmAm+DDYW43r4aLL0IpxZ2znjHkqMe9WUQGFW1z66zd5FWnbVJph+HPgdO38rC0bnaxlGp4EjsgLMVSp3JoveyRNFernNgur79/hEqApGaNw0wIdq+0uT1GIN7MzFitWaLYEXdbwgWmGYroXCzL4f2Jcxuz4jlsrvo+BhP36wRHKjmflq36j+D9bcg1E6QLT94auind7POTHnvh/uERvVKxGJdW6b4DpuHYiWo8s0EwK2PE2d3+7nXCH1iTxV7+2sDPmcMmqrPbGcrIH0UXR5bb7d2P64v4CgiynMUNx3AlK9JJBSm1czBQFKlMQ", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("www.tab.com.au", "179643557%7CMCIDTS%7C19777%7CMCMID%7C42816005962630671342879997247751714175%7CMCAAMLH-1709253617%7C8%7CMCAAMB-1709253617%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1708656017s%7CNONE%7CMCAID%7CNONE%7CMCCIDH%7C-394740951%7CvVersion%7C5.5.0", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("bm_mi", "7E78507F65544A6BEB911B6594252269~YAAQOU5haLLFDcqNAQAA1N1n0xaR8VACVmX0VDfQyabr26S83xcjsuZDySCB0Ba3KF8V7EBTrDO2I0q0LqCGVCXBIEpm4kqRUrsWAnZ2ku6pa3LWedJ/oPEPdqXXQz4MRQoqBv3xk/iKpQsEscblrzmKbEh1S+MoRjUsDv05k7V8/YypSTJlyfrhrA4XFBRJcyKThMUSgVmKmftm6lFi2Iv5X32BWk9tdClyDo1WcoR1gf3benLG+0dVsRJlJi3GhxxJh6NzUnbl5iuFh+p7VQtjevNJYtVX66OX773tSVIteUnl/rYsf5u/6dPm1F8FllaIkx0BtA==~1", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_cs_mk_aa", "0.1392149211434872_1708648818354", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("adcloud", "{%22_les_v%22:%22y%2Ctab.com.au%2C1708650618%22}", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_tguatd", "eyJzYyI6IihkaXJlY3QpIn0=", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_tgidts", "eyJzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwiY2kiOiJkODhmOTQ2ZS0xNGY2LTU2MDUtODJlOS02N2FiZjA0NDQ0MzIiLCJzaSI6IjBhYzA0OGMzLTliZjgtNTBkZC04ZmZhLTQzY2U3N2ZkNGQ5MCJ9", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_tglksd", "eyJzIjoiMGFjMDQ4YzMtOWJmOC01MGRkLThmZmEtNDNjZTc3ZmQ0ZDkwIiwic3QiOjE3MDg2NDg4MTg1MTUsInNvZCI6IihkaXJlY3QpIiwic29kdCI6MTcwMTMxNDI0MjY2NSwic29kcyI6Im8iLCJzb2RzdCI6MTcwMTMxNDI0MjY2NX0=", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_gid", "GA1.3.286042675.1708648819", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_ga", "GA1.3.1084240241.1701314243", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_tgsid", "eyJscGQiOiJ7XCJscHVcIjpcImh0dHBzOi8vd3d3LnRhYi5jb20uYXUlMkZcIixcImxwdFwiOlwiT25saW5lJTIwU3BvcnRzJTIwJTI2JTIwSG9yc2UlMjBSYWNpbmclMjBCZXR0aW5nJTIwQXVzdHJhbGlhJTIwJTdDJTIwVEFCXCIsXCJscHJcIjpcIlwifSIsInBzIjoiZjk5OWNmMDktNDlkOC00OTM3LTgyMmQtNzVkYWJhMzlhYjM1IiwicHZjIjoiMSIsInNjIjoiMGFjMDQ4YzMtOWJmOC01MGRkLThmZmEtNDNjZTc3ZmQ0ZDkwOi0xIiwiZWMiOiI0IiwicHYiOiIxIiwidGltIjoiMGFjMDQ4YzMtOWJmOC01MGRkLThmZmEtNDNjZTc3ZmQ0ZDkwOjE3MDg2NDg4MjE4MDU6LTEifQ==", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_cs_id", "8c885f2f-4ec7-a81f-a51d-0c7fbc4e5253.1701314242.13.1708649137.1708648818.1.1735478242634.1", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_cs_s", "4.0.0.1708650937050", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("mbox", "PC#4d529f94d644416b83c7f3f4420ea6c1.36_0#1771893939|session#e85cb33e18f1458fae30265336746b93#1708650999", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_uetsid", "1e537e60d1e411eeb9d3b9dc7b7b2ef3", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_uetvid", "fa7b46408f2e11ee94a259059b2087fb", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("bm_sv", "FA3401F77595CB3BFA48E9B3EB68D923~YAAQB1rWF90I/s+NAQAAvYR40xY/4Sg4442TcILWMORP16M+CUypxE/1Q8GoIPyAaR3Dw6QFQYepBzb6OzgAuHqjwVwgKffGPGlNSIakay0I67nwTvg22/n4RLbWrXHiKR1WUnyy0xS1ldQdlsQLEU9TbDrhJbIKxcuaoIvFSlWb13RSwFgigd6I77vhvfog4dYam2Prx93Z9S5AsP/Yt8mjF+UnQjukoloUsVSrB/tmd1rbngxGNFwKDA08xL7MVjU=~1", "/", ".tab.com.au")))
$session.Cookies.Add((New-Object System.Net.Cookie("_ga_ZMHBYSQ22N", "GS1.1.1708648818.12.1.1708649915.60.0.0", "/", ".tab.com.au")))
Invoke-WebRequest -UseBasicParsing -Uri "https://api.beta.tab.com.au/v1/tab-info-service/racing/dates/ThisRaceDate/meetings/R/ThisRaceCode/races/999?jurisdiction=VIC" `
-WebSession $session `
-Headers @{
"authority"="api.beta.tab.com.au"
  "method"="GET"
  "path"="/v1/tab-info-service/racing/dates/today/meetings?jurisdiction=VIC"
  "scheme"="https"
  "accept"="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
  "accept-encoding"="gzip, deflate, br"
  "accept-language"="en-US,en;q=0.9"
  "cache-control"="max-age=0"
  "dnt"="1"
  "if-none-match"="W/`"850d3337bda07909d530fde6a483487e`""
  "sec-ch-ua"="`"Not A(Brand`";v=`"99`", `"Google Chrome`";v=`"121`", `"Chromium`";v=`"121`""
  "sec-ch-ua-mobile"="?0"
  "sec-ch-ua-platform"="`"Windows`""
  "sec-fetch-dest"="document"
  "sec-fetch-mode"="navigate"
  "sec-fetch-site"="none"
  "sec-fetch-user"="?1"
  "upgrade-insecure-requests"="1"
} -OutFile Race.json

我的问题是如何使用所有这 30 个奇怪的元素来装饰代码 HTTPWebRequest，这些元素是使用代码中的 session.Cookies.Add 命令通过 Powershell 设置的，并在从浏览器运行时显示为请求标头中的项目（或作为Postman 中的curl 脚本）。

我尝试在代码中添加 CookieContainer 但无济于事，但也许我没有正确实现它。我过去从来不需要设置 cookie，所以我对如何实现它们的了解有限。

我不一定需要 VB.Net 中的解决方案。我可以改成C#。由于一些遗留代码，我只有 VB.Net，并且没有花时间将其全部更改为 C#

更新1 我现在可以通过添加一堆标头来获取此代码以获取响应，这与我在 Powershell 命令中找到的相同。

这是我的代码

'Using request As HttpWebRequest
        request = HttpWebRequest.Create(New Uri(Url))
        request.Pipelined = True
        request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
        request.Method = "GET"
        request.Timeout = 5000
        request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
        request.Headers.Add("authority", "api.beta.tab.com.au")
        request.Headers.Add("method", "GET")
        request.Headers.Add("scheme", "https")
        request.Headers.Add("accept-encoding", "gzip, deflate, br, zstd")
        request.Headers.Add("accept-language", "en-US,en;q=0.9")
        request.Headers.Add("cache-control", "max-age=0")
        request.Headers.Add("dnt", "1")

        request.AutomaticDecompression = DecompressionMethods.GZip
        request.CookieContainer = cookies
        response = request.GetResponse()

        strResponseStatus = CType(response, HttpWebResponse).StatusDescription
        dataStream = response.GetResponseStream()
        reader = New StreamReader(dataStream)
        responseFromServer = reader.ReadToEnd()

        reader.Close()
        dataStream.Close()
        response.Close()

这将成功运行并第一次返回响应，但我每 15 秒循环一次。有时下一个循环也可以工作，但有时则不行。有时连续 3 个循环有效，然后失败，下一个循环有效。没有一致性。

我做错了什么？我关闭了读取流，所以我认为不是这样的。我仅使用 HTTPWebRequest 的单个实例并将对象传递给此代码。

我同意它不像我最初怀疑的那样是 HTTP/2。它没有将超时设置为 5 秒，因为即使设置为 120 秒也会超时。当它起作用时，需要几毫秒。

当它失败时，我注意到 Pipelined 设置为 false，因此我将其设置为 true，但这似乎没有什么区别。

Answer 1

经过至少几天的尝试不同的选择，我找到了解决方案。所以我正在回答我自己的问题。

我发现，当您发送请求时，您需要提供一个 cookie 容器，以便在获取响应时接收 cookie。然后，您必须通过该 cookie 容器发送后续请求。

我的请求代码位于函数（VB .Net）中，每次调用此函数时我都会将 cookie 容器设置为新对象。

我修改了函数以将 cookie 容器作为参数，并在循环外仅创建一个 cookie 容器，每次都将其传递给函数。第一次调用该函数时，它会填充空的 cookie 容器，然后后续调用使用填充的容器。

这是代码

    Function CreateRequestSimple(ByVal sURL As String, ByVal sAuthority As String, ByRef request As HttpWebRequest, ByRef cookies2 As CookieContainer) As String
    Dim Url As String
    Dim dataStream As Stream = Nothing
    Dim response As HttpWebResponse = Nothing
    Dim strResponseStatus As String = ""
    Dim reader As StreamReader = Nothing
    Dim responseFromServer As String = ""
    Dim cookies As CookieContainer = New CookieContainer()
    Dim expiredcookie As Boolean


    Try

        request = HttpWebRequest.Create(New Uri(sURL))
        request.Pipelined = True 'I don't think this does anything
        request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
        request.Method = "GET"
        request.Timeout = 5000
        request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
        request.Headers.Add("authority", sAuthority)
        request.Headers.Add("method", "GET")
        request.Headers.Add("scheme", "https")
        request.Headers.Add("accept-encoding", "gzip, deflate, br, zstd")
        request.Headers.Add("accept-language", "en-US,en;q=0.9")
        request.Headers.Add("cache-control", "max-age=0")
        request.Headers.Add("dnt", "1")

        request.AutomaticDecompression = DecompressionMethods.GZip

        expiredcookie = IsAnyCookieExpired(cookies2, request.RequestUri)
        If (expiredcookie) Then
            request.CookieContainer = cookies
            cookies2 = cookies
        Else
            request.CookieContainer = cookies2
        End If
        response = request.GetResponse()

        strResponseStatus = CType(response, HttpWebResponse).StatusDescription
        dataStream = response.GetResponseStream()
        reader = New StreamReader(dataStream)
        responseFromServer = reader.ReadToEnd()

        reader.Close()
        dataStream.Close()
        response.Close()


    Catch ex As Exception
        Return ex.Message
    End Try
    Return responseFromServer
End Function
Function IsAnyCookieExpired(ByVal cookies As CookieContainer, ByVal uri As Uri) As Boolean
    Dim cookiecoll As New CookieCollection

    '        cookiecoll = cookies.

    For Each cookie As Cookie In cookies.GetCookies(uri)
        If (cookie.Expired) Then
            Return True
        End If
    Next

    Return False
End Function

我认为我不需要检查 cookie 是否过期，因为我认为每次调用此函数时过期都会更新。

我这样调用函数（伪代码，不是我的真实代码）

            Dim request As HttpWebRequest = Nothing
        Dim cookies As CookieContainer = New CookieContainer()
        Dim sResponse As String = ""
        Dim sURL As String = ""
        Dim sAuthority As String = "auth.com.au"

        For i = 0 To 100

            sURL = "http://someurl.com/extra"
            sResponse = CreateRequestSimple(sURL, sAuthority, request, cookies)
        Next i

感谢 jdweng 指出 cookie 是主要问题。

此外，该网站已更改为以 GZip 格式发送响应，因此我也必须处理该更改。再次感谢jdweng提出这个问题