Browse code
Added user agent to other call to requests.get
E L authored on 22/10/2015 00:07:19
Showing 1 changed files
Showing 1 changed files
... | ... |
@@ -24,7 +24,7 @@ class DefaultTitleGetter(object): |
24 | 24 |
canonicalLink = etree.xpath('//link[@rel="canonical"]/@href') |
25 | 25 |
if canonicalLink != []: |
26 | 26 |
canonicalLink = canonicalLink[0] |
27 |
- data = s.get(canonicalLink) |
|
27 |
+ data = s.get(canonicalLink, headers=self.user_agent) |
|
28 | 28 |
etree = lxml.html.fromstring(data.content.decode(data.encoding)) |
29 | 29 |
else: |
30 | 30 |
canonicalLink = url |