python - 使用 urllib2 检查响应

Question

我正在尝试使用 opencorporates api 通过增加页面计数器来访问页面。但问题是有时会存在无用的数据。例如，在下面的辖区代码 = ae_az 的网址中，我得到的网页仅显示以下内容:

{"api_version":"0.2","results":{"companies":[],"page":1,"per_page":26,"total_pages":0,"total_count":0}}

这在技术上是空的。如何检查此类数据并跳过此数据以进入下一个管辖区？

这是我的代码

import urllib2
import json,os

f = open('codes','r')
for line in f.readlines():
   id = line.strip('\n')
   url = 'http://api.opencorporates.com/v0.2/companies/search?q=&jurisdiction_code={0}&per_page=26&current_status=Active&page={1}?api_token=ab123cd45' 
   i = 0
   directory = id
   os.makedirs(directory)
   while True:
      i += 1
      req = urllib2.Request(url.format(id, i))
      print url.format(id,i)
      try:
        response = urllib2.urlopen(url.format(id, i))
      except urllib2.HTTPError, e:
        break
      content = response.read()
      fo = str(i) + '.json'    
      OUTFILE = os.path.join(directory, fo)
      with open(OUTFILE, 'w') as f:
        f.write(content)

Answer 1

解释您返回的响应(您已经知道它是 json)并检查您想要的数据是否存在。

...
content = response.read()
data = json.loads(content)
if not data.get('results', {}).get('companies'):
    break
...

这是您使用请求编写的代码并使用此处的答案。它远没有应有的强大或干净，但展示了您可能想要采取的路径。速率限制是一个猜测，似乎不起作用。请记住输入您的实际 API key 。

import json
import os
from time import sleep
import requests

url = 'http://api.opencorporates.com/v0.2/companies/search'
token = 'ab123cd45'
rate = 20  # seconds to wait after rate limited

with open('codes') as f:
    codes = [l.strip('\n') for l in f]


def get_page(code, page, **kwargs):
    params = {
        # 'api_token': token,
        'jurisdiction_code': code,
        'page': page,
    }
    params.update(kwargs)

    while True:
        r = requests.get(url, params=params)

        try:
            data = r.json()
        except ValueError:
            return None

        if 'error' in data:
            print data['error']['message']
            sleep(rate)
            continue

        return data['results']


def dump_page(code, page, data):
    with open(os.path.join(code, str(page) + '.json'), 'w') as f:
        json.dump(data, f)


for code in codes:
    try:
        os.makedirs(code)
    except os.error:
        pass

    data = get_page(code, 1)
    if data is None:
        continue

    dump_page(code, 1, data['companies'])

    for page in xrange(1, int(data.get('total_pages', 1))):
        data = get_page(code, page)
        if data is None:
            break

        dump_page(code, page, data['companies'])