73 lines
2.2 KiB
Plaintext
73 lines
2.2 KiB
Plaintext
|
|
def fetch_url_requests(url):
|
|
print("fetch_url_requests: Sending request with requests...")
|
|
# squid3
|
|
squid3_proxies = {
|
|
"http": "http://127.0.0.1:3128",
|
|
"https": "http://127.0.0.1:3128",
|
|
}
|
|
# polipo
|
|
polipo_proxies = {
|
|
# "http": "http://127.0.0.1:8123",
|
|
# "https": "http://127.0.0.1:8123",
|
|
"http": "http://127.0.0.1:5566",
|
|
"https": "http://127.0.0.1:5566",
|
|
}
|
|
print("fetch_url_requests: url: " + url)
|
|
try:
|
|
r = requests.get(url, proxies=polipo_proxies)
|
|
text = r.text
|
|
# print("fetch_url_requests: text: " + text)
|
|
return text
|
|
except(TypeError, BadStatusLine):
|
|
fetch_url_requests(url)
|
|
OR
|
|
except httplib.BadStatusLine:
|
|
fetch_url_requests(url)
|
|
OR
|
|
except TypeError:
|
|
fetch_url_requests(url)
|
|
|
|
except (socket.error, httplib.BadStatusLine):
|
|
----------------
|
|
try:
|
|
if not doc or not doc['coordinates']:
|
|
xxxxxxxxx
|
|
else:
|
|
xxxxxxxx
|
|
|
|
except (ValueError, geocoders.google.GQueryError):
|
|
pass
|
|
----------------
|
|
|
|
|
|
fetch_url_requests: url: http://solicitors.lawsociety.org.uk/person/233059/
|
|
Traceback (most recent call last):
|
|
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 331, in _make_request
|
|
httplib_response = conn.getresponse(buffering=True)
|
|
TypeError: getresponse() got an unexpected keyword argument 'buffering'
|
|
|
|
----------------------
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
File "/usr/lib/python3.4/http/client.py", line 321, in _read_status
|
|
raise BadStatusLine(line)
|
|
http.client.BadStatusLine: ''
|
|
|
|
--------------
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
File "/usr/lib/python3.4/http/client.py", line 321, in _read_status
|
|
raise BadStatusLine(line)
|
|
urllib3.exceptions.ProtocolError: ('Connection aborted.', BadStatusLine("''",))
|
|
|
|
---------------
|
|
|
|
During handling of the above exception, another exception occurred:
|
|
|
|
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 407, in send
|
|
raise ConnectionError(err, request=request)
|
|
requests.exceptions.ConnectionError: ('Connection aborted.', BadStatusLine("''",))
|