(defpackage :fwoar.lisp-sandbox.reddit-dataset-creator
   (:use :cl )
   (:export ))
 (in-package :fwoar.lisp-sandbox.reddit-dataset-creator)
 
 (defun get-data (subreddit)
   (alimenta:to-feed
    (fw.lu:closing
        (plump:parse
         (drakma:http-request (puri:merge-uris (format nil "~(~a~).rss?limit=100" subreddit)
                                               "https://www.reddit.com/r/")
                              :want-stream t)))))
 
 (defun get-data-json (subreddit)
   (fw.lu:closing
       (yason:parse
        (drakma:http-request (puri:merge-uris (format nil "~(~a~).json?limit=100" subreddit)
                                              "https://www.reddit.com/r/")
                             :want-stream t))))
 
 (defun subreddit-json-image-urls (subreddit &optional (fetch 'get-data-json-cached))
   (data-lens.transducers:transduce
    (data-lens:•
     (data-lens.transducers:mapping
      (data-lens:key "data"))
     #+nil
     (data-lens.transducers:mapping
      (lambda (it)
        (format t "~&~s~%" (gethash "url" it))
        it))
     (data-lens.transducers:mapping
      (data-lens:• (lambda (it)
                     (fw.lu:dive '("source" "url")
                                 it))
                   (data-lens:applicable-when (data-lens:element 0)
                                              (complement #'null))
                   (data-lens:sorted '< :key (data-lens:key "width"))
                   (lambda (it)
                     (fw.lu:dive '("preview" "images")
                                 it))))
     (data-lens.transducers:filtering 'identity))
    'data-lens.transducers:list-builder
    (fw.lu:dive '("data" "children")
                (funcall fetch subreddit))))
 
 (defun subreddit-image-urls (subreddit &optional (fetch 'get-data-cached))
   (data-lens.transducers:transduce
    (data-lens:•
     (data-lens.transducers:mapping
      (data-lens:•
       (lambda (it)
         (lquery:$ (initialize it) "a" (attr "href")))
       'alimenta:content))
     (data-lens.transducers:catting)
     (data-lens.transducers:filtering
      (data-lens:regex-match "[.](jpe?g|png)$")))
    'data-lens.transducers:list-builder
    (funcall fetch subreddit)))
 
 (defvar *reset-cache* nil)
 (defun get-data-cached (subreddit)
   (let ((cache (load-time-value (make-hash-table))))
     (when *reset-cache*
       (clrhash cache)
       (setf *reset-cache* nil))
     (alexandria:ensure-gethash subreddit
                                cache
                                (get-data subreddit))))
 
 (defun get-data-json-cached (subreddit)
   (let ((cache (load-time-value (make-hash-table))))
     (when *reset-cache*
       (clrhash cache)
       (setf *reset-cache* nil))
     (alexandria:ensure-gethash subreddit
                                cache
                                (get-data-json subreddit))))
 
 
 (defun dump-stream (input output &optional fn)
   (with-open-stream (echo (make-echo-stream input output))
     (loop with buffer-length = 1000
           with buffer = (make-array buffer-length :element-type 'character)
           for read-chars = (read-sequence buffer echo)
           do (when fn
                (funcall fn buffer buffer-length))
              (unless (< read-chars buffer-length)
                (return nil)))))
 
 (defun connect-streams (input output &key (background t) fn)
   "This reads from input and writes output until the end of input is found."
   (dump-stream input output fn))
 
 (defun store-image (base category url)
   (let* ((uri (puri:parse-uri url))
          (fn (pathname-name
               (parse-namestring
                (puri:uri-path uri)))))
     (alexandria:with-output-to-file (s (ensure-directories-exist
                                         (merge-pathnames (make-pathname
                                                           :directory (list :relative
                                                                            (string category))
                                                           :name fn)
                                                          (parse-namestring base))))
       (fw.lu:closing
           (connect-streams (drakma:http-request url :want-stream t)
                            s
                            :background nil)))))
 
 (format *standard-output* "~&~s ~s ~s ~s ~s ~s~%"
         SB-EXT:*CORE-PATHNAME*
         SB-EXT:*RUNTIME-PATHNAME*
         asdf/user::*nil-pathname*
         *COMPILE-FILE-PATHNAME*
         *LOAD-PATHNAME*
         swank/sbcl::*buffer-name*
         )
 
 #+(or)
 (asdf:initialize-source-registry
  (fwoar.git-systems:define-dir-deps ()
      (:git"alimenta"              "git@git.fiddlerwoaroof.com:u/edwlan/alimenta.git"              "master")
    (:git"alimenta-feed-archive" "git@git.fiddlerwoaroof.com:u/edwlan/alimenta-feed-archive.git" "master")
    (:git"data-lens"             "git@git.fiddlerwoaroof.com:data-lens.git"                      "master")
    (:git"collection-class"      "git@git.fiddlerwoaroof.com:u/edwlan/collection-class.git"      "master")))