Web抓取页面表Python

Question

我正在尝试从网站上刮取一张表，但是我对Python不太熟悉。网址为here我想以data.frame格式获取表格，以便继续进行分析。

这里是我测试过的代码：

from lxml import html

import requests

from bs4 import BeautifulSoup

from urllib.request import urlopen, Request

url="https://www.wunderground.com/history/daily/gr/almyros/LGBL/date/2017-4-4"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}

req = Request(url=url1, headers=headers) 

html = urlopen(req).read() 

print(html)

soup = BeautifulSoup(html)

type(soup)

print(soup)

p=soup.find_all('thead')

k=soup.find_all('tbody')

print(k)

我不知道如何获得每日观察表

Answer 1

尝试一下：

import pandas as pd
from pandas.io.json import json_normalize
import re
import requests

url="https://www.wunderground.com/history/daily/gr/almyros/LGBL/date/2017-4-4"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}

with requests.Session() as s:
    response = s.get(url, headers=headers)
    p = re.compile(r'point\?apiKey=([a-zA-z0-9]*)&', re.MULTILINE)
    apiKey = re.findall(p, response.text)[0]

    url = 'https://api.weather.com/v1/location/LGBL:9:GR/observations/historical.json'
    payload = {
            'apiKey': apiKey,
            'startDate': '20170404',
            'endDate': '20170404'}

    jsonData = s.get(url, headers=headers, params=payload).json()
df = json_normalize(jsonData['observations'])

输出：

print (df.to_string())
   blunt_phrase        class clds day_ind  dewPt  expire_time_gmt  feels_like  gust  heat_index  icon_extd   key max_temp min_temp obs_id          obs_name precip_hrly precip_total  pressure pressure_desc pressure_tend primary_swell_direction primary_swell_height primary_swell_period primary_wave_height primary_wave_period qualifier qualifier_svrty   rh secondary_swell_direction secondary_swell_height secondary_swell_period snow_hrly  temp terse_phrase   uv_desc  uv_index  valid_time_gmt  vis water_temp  wc  wdir wdir_cardinal  wspd  wx_icon      wx_phrase
0          None  observation  OVC       N     45       1491263400          44  None          46       2600  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    46         None       Low         0      1491256200    6       None  44   260             W     5       26         Cloudy
1          None  observation  OVC       N     45       1491267000          44  None          46       2600  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    46         None       Low         0      1491259800    6       None  44   250           WSW     6       26         Cloudy
2          None  observation  OVC       N     43       1491270600          40  None          45       2600  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    45         None       Low         0      1491263400    6       None  40   250           WSW     8       26         Cloudy
3          None  observation  OVC       N     41       1491274200          39  None          43       2600  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    43         None       Low         0      1491267000    6       None  39   250           WSW     6       26         Cloudy
4          None  observation  OVC       N     43       1491277800          39  None          43       2600  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None  100                      None                   None                   None      None    43         None       Low         0      1491270600    6       None  39   250           WSW     7       26         Cloudy
5          None  observation  SCT       N     39       1491281400          36  None          41       2900  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    41         None       Low         0      1491274200    6       None  36   250           WSW     8       29  Partly Cloudy
6          None  observation  FEW       N     39       1491283200          36  None          41       3300  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    41         None       Low         0      1491276000    6       None  36   260             W     8       33           Fair
7          None  observation  FEW       D     39       1491285000          36  None          41       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    41         None       Low         0      1491277800    6       None  36   250           WSW     7       34           Fair
8          None  observation  FEW       D     39       1491286800          37  None          41       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    41         None       Low         1      1491279600    6       None  37   250           WSW     6       34           Fair
9          None  observation  FEW       D     39       1491288600          36  None          41       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    41         None       Low         1      1491281400    6       None  36   250           WSW     7       34           Fair
10         None  observation  FEW       D     41       1491290400          39  None          43       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   93                      None                   None                   None      None    43         None       Low         1      1491283200    6       None  39   260             W     6       34           Fair
11         None  observation  FEW       D     45       1491292200          46  None          48       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   87                      None                   None                   None      None    48         None       Low         1      1491285000    6       None  46   240           WSW     6       34           Fair
12         None  observation  FEW       D     46       1491294000          50  None          50       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   87                      None                   None                   None      None    50         None       Low         1      1491286800    6       None  50   270             W     2       34           Fair
13         None  observation  FEW       D     46       1491295800          54  None          54       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   77                      None                   None                   None      None    54         None       Low         2      1491288600    6       None  54   250           WSW     2       34           Fair
14         None  observation  FEW       D     48       1491297600          57  None          57       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   72                      None                   None                   None      None    57         None  Moderate         3      1491290400    6       None  57   200           SSW     2       34           Fair
15         None  observation  FEW       D     48       1491299400          57  None          57       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   72                      None                   None                   None      None    57         None  Moderate         3      1491292200    6       None  57    90             E     3       34           Fair
16         None  observation  FEW       D     48       1491301200          57  None          57       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   72                      None                   None                   None      None    57         None  Moderate         4      1491294000    6       None  57   100             E     3       34           Fair
17         None  observation  FEW       D     48       1491303000          59  None          59       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   67                      None                   None                   None      None    59         None  Moderate         5      1491295800    6       None  59    80             E     3       34           Fair
18         None  observation  SCT       D     48       1491304800          59  None          59       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   67                      None                   None                   None      None    59         None  Moderate         4      1491297600    6       None  59    80             E     3       30  Partly Cloudy
19         None  observation  SCT       D     48       1491306600          61  None          61       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   63                      None                   None                   None      None    61         None  Moderate         5      1491299400    6       None  61    70           ENE     6       30  Partly Cloudy
20         None  observation  SCT       D     50       1491308400          61  None          61       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   68                      None                   None                   None      None    61         None  Moderate         5      1491301200    6       None  61    60           ENE     5       30  Partly Cloudy
21         None  observation  SCT       D     48       1491310200          61  None          61       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     30.00          None          None                    None                 None                 None                None                None      None            None   63                      None                   None                   None      None    61         None  Moderate         5      1491303000    6       None  61    80             E     6       30  Partly Cloudy
22         None  observation  SCT       D     50       1491312000          61  None          61       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   68                      None                   None                   None      None    61         None  Moderate         5      1491304800    6       None  61    70           ENE     6       30  Partly Cloudy
23         None  observation  SCT       D     50       1491313800          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   63                      None                   None                   None      None    63         None  Moderate         4      1491306600    6       None  63    70           ENE     6       30  Partly Cloudy
24         None  observation  SCT       D     50       1491315600          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   63                      None                   None                   None      None    63         None  Moderate         4      1491308400    6       None  63    70           ENE     6       30  Partly Cloudy
25         None  observation  SCT       D     50       1491317400          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   63                      None                   None                   None      None    63         None  Moderate         3      1491310200    6       None  63   100             E     7       30  Partly Cloudy
26         None  observation  SCT       D     52       1491319200          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   68                      None                   None                   None      None    63         None  Moderate         3      1491312000    6       None  63    80             E     7       30  Partly Cloudy
27         None  observation  SCT       D     52       1491321000          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   68                      None                   None                   None      None    63         None       Low         2      1491313800    6       None  63    90             E     7       30  Partly Cloudy
28         None  observation  SCT       D     52       1491322800          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   68                      None                   None                   None      None    63         None       Low         2      1491315600    6       None  63    90             E     8       30  Partly Cloudy
29         None  observation  SCT       D     52       1491324600          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   68                      None                   None                   None      None    63         None       Low         1      1491317400    6       None  63    70           ENE     8       30  Partly Cloudy
30         None  observation  SCT       D     50       1491326400          63  None          63       3000  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   63                      None                   None                   None      None    63         None       Low         1      1491319200    6       None  63   100             E    10       30  Partly Cloudy
31         None  observation  FEW       D     52       1491328200          61  None          61       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.94          None          None                    None                 None                 None                None                None      None            None   72                      None                   None                   None      None    61         None       Low         1      1491321000    6       None  61   110           ESE    14       34           Fair
32         None  observation  FEW       D     54       1491330000          59  None          59       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   82                      None                   None                   None      None    59         None       Low         1      1491322800    6       None  59   110           ESE    10       34           Fair
33         None  observation  FEW       D     54       1491331800          59  None          59       3400  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   82                      None                   None                   None      None    59         None       Low         1      1491324600    6       None  59   120           ESE    12       34           Fair
34         None  observation  FEW       N     54       1491333600          57  None          57       3300  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   88                      None                   None                   None      None    57         None       Low         0      1491326400    6       None  57   120           ESE    10       33           Fair
35         None  observation  FEW       N     54       1491335400          57  None          57       3300  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   88                      None                   None                   None      None    57         None       Low         0      1491328200    6       None  57   110           ESE    13       33           Fair
36         None  observation  FEW       N     54       1491339000          55  None          55       3300  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   94                      None                   None                   None      None    55         None       Low         0      1491331800    6       None  55    90             E     5       33           Fair
37         None  observation  FEW       N     52       1491342600          55  None          55       3300  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   88                      None                   None                   None      None    55         None       Low         0      1491335400    6       None  55   130            SE     5       33           Fair
38         None  observation  FEW       N     48       1491346200          50  None          50       3300  LGBL     None     None   LGBL  Volos/Nea Anghia        None         None     29.97          None          None                    None                 None                 None                None                None      None            None   94                      None                   None                   None      None    50         None       Low         0      1491339000    6       None  50   270             W     2       33           Fair