git.fiddlerwoaroof.com
tools/file-indexer.lisp
ad68ec3d
 #+fw.dump
 (eval-when (:compile-toplevel :load-toplevel :execute)
   (load "~/quicklisp/setup.lisp")
   (require :asdf))
 
 #+fw.dump
 (progn
   (asdf:load-asd
    (asdf:system-relative-pathname :net.didierverna.clon "termio/net.didierverna.clon.termio.asd"))
   (require :uiop))
 
 #+fw.dump
 (ql:quickload '(:net.didierverna.clon :ironclad :sqlite))
 
 (defpackage :fwoar.file-indexer
   (:use :cl)
   (:export ))
 (in-package :fwoar.file-indexer)
 (defvar *synopsis*
   (net.didierverna.clon:defsynopsis (:postfix "DB FILES..." :make-default nil)
     (flag :short-name "h" :long-name "help")))
 
df7db01a
 
ad68ec3d
 (defun main ()
   (let* ((context (net.didierverna.clon:make-context :synopsis *synopsis*))
6e4abe23
          (net.didierverna.clon:*context* context))
ad68ec3d
     (cond ((net.didierverna.clon:getopt :context context
                                         :long-name "help")
            (net.didierverna.clon:help))
           (t
            (destructuring-bind (db-fn . files)
                (net.didierverna.clon:remainder :context context)
              (sqlite:with-open-database (db db-fn)
df7db01a
                (sqlite:execute-non-query db "pragma journal_mode=wal")
ad68ec3d
                (sqlite:execute-non-query
                 db
                 "create table if not exists
df7db01a
                   files_shasums (file text,
                                  shasum text,
f2f3ba89
                                  size bigint,
                                  count bigint,
                                  ts datetime default current_timestamp)")
ad68ec3d
                (sqlite:execute-non-query
                 db
                 "create unique index if not exists shasums_files_unique_assuc on files_shasums(file,shasum)")
df7db01a
                (loop for raw-file in files
                      for file = (uiop:parse-native-namestring raw-file)
ad68ec3d
                      do
f2f3ba89
                         (when (< (random 1000000)
                                  100)
                           (format *error-output* "~&processed: ~a~%" (uiop:native-namestring file)))
ad68ec3d
                         (sqlite:with-transaction db
df7db01a
                           (with-open-file (s file :element-type '(unsigned-byte 8))
                             (let* ((sum (ironclad:byte-array-to-hex-string
                                          (ironclad:digest-file :sha256 file)))
                                    (length (file-length s)))
                               (sqlite:execute-single db
f2f3ba89
                                                      "insert into files_shasums (file,shasum,size,count)
                                                       values (?,?,?, 1)
                                                       on conflict do update set count = (count + 1),
                                                                                 ts = current_timestamp"
d32bda99
                                                      (uiop:native-namestring (truename file))
df7db01a
                                                      sum
f2f3ba89
                                                      length)))))))))))
ad68ec3d
 
 (defun dump ()
   (setf net.didierverna.clon:*context* nil
         *features* (remove :fw.dump *features*))
   (net.didierverna.clon:dump "file-indexer" main))