python爬虫提取数据

当使用python我们发送requests请求时,获取响应之后,

可能存在多种不同类型的响应内容;而且很多时候,我们只需要响应内容中的一部分数据

我们如何提取这些内容呢?

首先我们来多response响应来做个分类

结构化

非结构化

  • 结构化的响应内容

    • json字符串(高频出现使用很多)

    • 可以使用re、json、jsonpath等模块来提取特定数据

    • json字符串的例子如下图

  • [
        {
            "ASUB_CODE": "1001",
            "ASUB_NAME": "库存现金",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 4888,
            "CREDIT": 0,
            "TOTAL_DEBIT": 4888,
            "TOTAL_CREDIT": 0
        },
        {
            "ASUB_CODE": "1122",
            "ASUB_NAME": "应收账款",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 2880,
            "CREDIT": 0,
            "TOTAL_DEBIT": 2880,
            "TOTAL_CREDIT": 0
        },
        {
            "ASUB_CODE": "1405",
            "ASUB_NAME": "库存商品",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 660,
            "CREDIT": 0,
            "TOTAL_DEBIT": 660,
            "TOTAL_CREDIT": 0
        },
        {
            "ASUB_CODE": "2202",
            "ASUB_NAME": "应付账款",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 0,
            "CREDIT": 745.8,
            "TOTAL_DEBIT": 0,
            "TOTAL_CREDIT": 745.8
        },
        {
            "ASUB_CODE": "2203",
            "ASUB_NAME": "预收账款",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 0,
            "CREDIT": 4888,
            "TOTAL_DEBIT": 0,
            "TOTAL_CREDIT": 4888
        },
        {
            "ASUB_CODE": "2221",
            "ASUB_NAME": "应交税费",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 85.8,
            "CREDIT": 331.32,
            "TOTAL_DEBIT": 245.52,
            "TOTAL_CREDIT": 0
        },
        {
            "ASUB_CODE": "2221001",
            "ASUB_NAME": "应交增值税",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 85.8,
            "CREDIT": 331.32,
            "TOTAL_DEBIT": 245.52,
            "TOTAL_CREDIT": 0
        },
        {
            "ASUB_CODE": "222100101",
            "ASUB_NAME": "进项税额",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 85.8,
            "CREDIT": 0,
            "TOTAL_DEBIT": 85.8,
            "TOTAL_CREDIT": 0
        },
        {
            "ASUB_CODE": "222100105",
            "ASUB_NAME": "销项税额",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 0,
            "CREDIT": 331.32,
            "TOTAL_DEBIT": 0,
            "TOTAL_CREDIT": 331.32
        },
        {
            "ASUB_CODE": "5001",
            "ASUB_NAME": "主营业务收入",
            "INITIAL_DEBIT": 0,
            "INITIAL_CREDIT": 0,
            "DEBIT": 0,
            "CREDIT": 2548.68,
            "TOTAL_DEBIT": 0,
            "TOTAL_CREDIT": 2548.68
        }
    ]

    xml字符串

    • 可以使用re、lxml等模块来提取特定数据

    • xml字符串的例子如下

<bookstore>
<book category="COOKING">
  <title lang="en">Everyday Italian</title> 
  <author>Giada De Laurentiis</author> 
  <year>2005</year> 
  <price>30.00</price> 
</book>
<book category="CHILDREN">
  <title lang="en">Harry Potter</title> 
  <author>J K. Rowling</author> 
  <year>2005</year> 
  <price>29.99</price> 
</book>
<book category="WEB">
  <title lang="en">Learning XML</title> 
  <author>Erik T. Ray</author> 
  <year>2003</year> 
  <price>39.95</price> 
</book>
</bookstore>