xinbiancheng.cn

python爬虫提取数据

当使用python我们发送requests请求时，获取响应之后，

可能存在多种不同类型的响应内容；而且很多时候，我们只需要响应内容中的一部分数据

我们如何提取这些内容呢？

首先我们来多response响应来做个分类

结构化

非结构化

  • 结构化的响应内容

    • json字符串(高频出现使用很多)

    • 可以使用re、json、jsonpath等模块来提取特定数据

    • json字符串的例子如下图

  • [
    {
        "ASUB_CODE": "1001",
        "ASUB_NAME": "库存现金",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 4888,
        "CREDIT": 0,
        "TOTAL_DEBIT": 4888,
        "TOTAL_CREDIT": 0
    },
    {
        "ASUB_CODE": "1122",
        "ASUB_NAME": "应收账款",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 2880,
        "CREDIT": 0,
        "TOTAL_DEBIT": 2880,
        "TOTAL_CREDIT": 0
    },
    {
        "ASUB_CODE": "1405",
        "ASUB_NAME": "库存商品",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 660,
        "CREDIT": 0,
        "TOTAL_DEBIT": 660,
        "TOTAL_CREDIT": 0
    },
    {
        "ASUB_CODE": "2202",
        "ASUB_NAME": "应付账款",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 0,
        "CREDIT": 745.8,
        "TOTAL_DEBIT": 0,
        "TOTAL_CREDIT": 745.8
    },
    {
        "ASUB_CODE": "2203",
        "ASUB_NAME": "预收账款",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 0,
        "CREDIT": 4888,
        "TOTAL_DEBIT": 0,
        "TOTAL_CREDIT": 4888
    },
    {
        "ASUB_CODE": "2221",
        "ASUB_NAME": "应交税费",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 85.8,
        "CREDIT": 331.32,
        "TOTAL_DEBIT": 245.52,
        "TOTAL_CREDIT": 0
    },
    {
        "ASUB_CODE": "2221001",
        "ASUB_NAME": "应交增值税",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 85.8,
        "CREDIT": 331.32,
        "TOTAL_DEBIT": 245.52,
        "TOTAL_CREDIT": 0
    },
    {
        "ASUB_CODE": "222100101",
        "ASUB_NAME": "进项税额",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 85.8,
        "CREDIT": 0,
        "TOTAL_DEBIT": 85.8,
        "TOTAL_CREDIT": 0
    },
    {
        "ASUB_CODE": "222100105",
        "ASUB_NAME": "销项税额",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 0,
        "CREDIT": 331.32,
        "TOTAL_DEBIT": 0,
        "TOTAL_CREDIT": 331.32
    },
    {
        "ASUB_CODE": "5001",
        "ASUB_NAME": "主营业务收入",
        "INITIAL_DEBIT": 0,
        "INITIAL_CREDIT": 0,
        "DEBIT": 0,
        "CREDIT": 2548.68,
        "TOTAL_DEBIT": 0,
        "TOTAL_CREDIT": 2548.68
    }
]

    xml字符串

    • 可以使用re、lxml等模块来提取特定数据

    • xml字符串的例子如下

<bookstore>
<book category="COOKING">
  <title lang="en">Everyday Italian</title> 
  <author>Giada De Laurentiis</author> 
  <year>2005</year> 
  <price>30.00</price> 
</book>
<book category="CHILDREN">
  <title lang="en">Harry Potter</title> 
  <author>J K. Rowling</author> 
  <year>2005</year> 
  <price>29.99</price> 
</book>
<book category="WEB">
  <title lang="en">Learning XML</title> 
  <author>Erik T. Ray</author> 
  <year>2003</year> 
  <price>39.95</price> 
</book>
</bookstore>