16cdb16d |
(declaim (optimize (speed 0) (safety 3) (debug 3)))
(in-package :alimenta.pull-feed)
|
58a15352 |
(defmacro setup-libraries-for-feeds (&body body)
`(let ((plump:*tag-dispatchers* plump:*xml-tags*)
|
2cad3567 |
(drakma:*drakma-default-external-format* :utf-8)
(drakma:*text-content-types*
|
fa63b386 |
(pairlis '("application" "application" "application" "application")
'("atom+xml" "rss+xml" "xml" "rdf+xml")
|
2cad3567 |
drakma:*text-content-types*)))
|
58a15352 |
,@body))
|
a16882ba |
(defvar *user-agent* "alimenta/0.0")
(defun call-with-user-agent (user-agent cb &rest args)
(let ((*user-agent* user-agent))
(apply cb args)))
(defun let-bind-special-var-macro-body (var value body)
`(let ((,var ,value))
,@body))
(defmacro with-user-agent ((user-agent) &body body)
(let-bind-special-var-macro-body '*user-agent* user-agent body))
|
16cdb16d |
(defun fetch-doc-from-url (url)
|
5ab3e57a |
(setup-libraries-for-feeds
|
303fb089 |
(let ((data (drakma:http-request url :user-agent *user-agent* :decode-content t)))
(when (and (not (stringp data))
(vectorp data))
(setf data (babel:octets-to-string data)))
|
379b0b37 |
(plump:parse data))))
|
16cdb16d |
|
5ab3e57a |
(define-condition fetch-error (error) ())
|
16cdb16d |
(define-condition feed-ambiguous (fetch-error) ((choices :initarg :choices :initform nil)))
(define-condition no-feed (fetch-error) ((url :initarg :url :initform nil)))
|
5ab3e57a |
(defun skip-feed (&optional condition)
(when-let ((restart (find-restart 'skip-feed condition)))
(invoke-restart restart)))
|
16cdb16d |
(defun feed-ambiguous (choices)
(error 'feed-ambiguous
|
2cad3567 |
:choices choices))
|
16cdb16d |
(defun no-feed (url)
(cerror "Skip this feed" 'no-feed :url url))
(defun fetch-feed-from-url (url &key type)
|
58a15352 |
(setup-libraries-for-feeds
|
2cad3567 |
(let* ((feeds (alimenta.discover:discover-feed
(drakma:http-request url
:user-agent *user-agent*
:decode-content t)))
(feeds (if type (remove-if-not (lambda (x) (eql type (car x)))
feeds)
feeds)))
|
5ab3e57a |
(if (not feeds)
|
2cad3567 |
(no-feed url)
(fetch-doc-from-url
|
c24253ba |
(cdar
|
2cad3567 |
(restart-case
(if (cdr feeds) (feed-ambiguous feeds) feeds)
(take-first-feed nil
:report (lambda (s) (format s "Take the first feed"))
feeds)
(take-nth-feed (n)
:report (lambda (s) (format s "Take the nth feed"))
(list (elt feeds n)))
(select-feed (selector)
:report (lambda (s) (format s "Provide a function to select the right feed"))
(find-if selector feeds)))))))))
|
16cdb16d |
|
379b0b37 |
(defmacro with-retry ((retry-message &rest args) action &body other-restarts)
`(loop (restart-case (return ,action)
|
2cad3567 |
(retry ()
:report (lambda (s) (format s ,retry-message ,@args)))
,@other-restarts)))
|
379b0b37 |
|
58a15352 |
(defun pull-feed (url &key detect type)
|
379b0b37 |
(with-retry ("Retry fetching feed ~a" url)
|
c24253ba |
(to-feed
(if detect
(fetch-feed-from-url url)
(fetch-doc-from-url url))
:type type)
|
379b0b37 |
(skip-feed () (return)
|
2cad3567 |
:report (lambda (s) (format s "Skip fetching feed ~a" url)))))
|