Browse code
various changes
E L authored on 02/12/2015 23:36:08
Showing 2 changed files
Showing 2 changed files
... | ... |
@@ -22,10 +22,26 @@ class DefaultTitleGetter(object): |
22 | 22 |
etree = lxml.html.fromstring(data.content.decode(data.encoding)) |
23 | 23 |
|
24 | 24 |
canonicalLink = etree.xpath('//link[@rel="canonical"]/@href') |
25 |
+ oetree = etree |
|
25 | 26 |
if canonicalLink != []: |
26 | 27 |
canonicalLink = canonicalLink[0] |
27 |
- data = s.get(canonicalLink, headers=self.user_agent) |
|
28 |
- etree = lxml.html.fromstring(data.content.decode(data.encoding)) |
|
28 |
+ try: |
|
29 |
+ data = s.get(canonicalLink, headers=self.user_agent) |
|
30 |
+ etree = lxml.html.fromstring(data.content.decode(data.encoding)) |
|
31 |
+ except requests.exceptions.MissingSchema: |
|
32 |
+ nscheme, nnetloc, npath, nparams, nquery, nfragment = urlparse.urlparse(canonicalLink) |
|
33 |
+ if nscheme == '': |
|
34 |
+ nscheme = scheme |
|
35 |
+ if nnetloc == '': |
|
36 |
+ nnetloc = netloc |
|
37 |
+ canonicalLink = urlparse.urlunparse((nscheme, nnetloc, npath, nparams, nquery, nfragment)) |
|
38 |
+ try: |
|
39 |
+ data = s.get(canonicalLink, headers=self.user_agent) |
|
40 |
+ etree = lxml.html.fromstring(data.content.decode(data.encoding)) |
|
41 |
+ except IOError: |
|
42 |
+ etree = oetree |
|
43 |
+ except IOError: |
|
44 |
+ etree = oetree |
|
29 | 45 |
else: |
30 | 46 |
canonicalLink = url |
31 | 47 |
|
... | ... |
@@ -22,7 +22,7 @@ |
22 | 22 |
<link rel="icon" type="image/png" href="/images/icons/android-chrome-192x192.png?v=all2jv2L7Q" sizes="192x192"> |
23 | 23 |
<link rel="icon" type="image/png" href="/images/icons/favicon-16x16.png?v=all2jv2L7Q" sizes="16x16"> |
24 | 24 |
<link rel="manifest" href="/images/icons/manifest.json?v=all2jv2L7Q"> |
25 |
- <link rel="mask-icon" href="/images/icons/safari-pinned-tab.svg?v=all2jv2L7Q" color="#8eb7cf"> |
|
25 |
+ <link rel="mask-icon" href="/images/icons/safari-pinned-tab.svg" color="#8eb7cf"> |
|
26 | 26 |
<link rel="shortcut icon" href="/images/icons/favicon.ico?v=all2jv2L7Q"> |
27 | 27 |
<meta name="msapplication-TileColor" content="#da532c"> |
28 | 28 |
<meta name="msapplication-TileImage" content="/images/icons/mstile-144x144.png?v=all2jv2L7Q"> |