git.fiddlerwoaroof.com
Browse code

feat: improve garagiste puller

Ed L authored on 31/12/2019 05:29:29
Showing 1 changed files
... ...
@@ -3,6 +3,195 @@
3 3
   (:export ))
4 4
 (in-package :garagiste-app)
5 5
 
6
+(defclass recording-datastore ()
7
+  ((%record :accessor record
8
+            :initform (make-array 10
9
+                                  :adjustable t
10
+                                  :fill-pointer 0))))
11
+(defclass jsonapi-datastore ()
12
+  ((%database :reader database
13
+              :initform (make-hash-table))))
14
+(defgeneric add-object (store object))
15
+(defgeneric get-object (store type id))
16
+(defgeneric ingest-jsonapi (store object))
17
+(defgeneric stored-types (store))
18
+
19
+(defun object (store type id)
20
+  (get-object store type id))
21
+
22
+(defun (setf object) (new-value store type id)
23
+  (let ((tmp (alexandria:copy-hash-table new-value)))
24
+    (setf (gethash "id" tmp) id
25
+          (gethash "type" tmp) type)
26
+    (add-object store tmp)))
27
+
28
+(defclass batch ()
29
+  ((%id :accessor id)
30
+   (%shipments :accessor shipments-a)))
31
+(defmethod add-object ((store jsonapi-datastore) (object batch))
32
+  (setf (object :batches (id object))
33
+        (alexandria:alist-hash-table
34
+         `(("relationships" . ,(mapcar (lambda (it)
35
+                                         (alexandria:alist-hash-table
36
+                                          `(("type" . "shipments")
37
+                                            ("id" . ,it))))
38
+                                       (shipments-a object)))))))
39
+
40
+(defun normalize-type (type)
41
+  (declare (optimize debug))
42
+  (let ((normalize-char (lambda (ch)
43
+                          (case ch
44
+                            (#\_ #\-)
45
+                            (t (if (lower-case-p ch)
46
+                                   (char-upcase ch)
47
+                                   (char-downcase ch)))))))
48
+    (nth-value
49
+     0 (etypecase type
50
+         (simple-string (intern (map 'simple-string
51
+                                     normalize-char
52
+                                     type)
53
+                                :keyword))
54
+         (string (intern (map 'simple-string
55
+                              normalize-char
56
+                              type)
57
+                         :keyword))
58
+         (keyword type)))))
59
+
60
+(defmacro let-lazy ((&rest bindings) &body body)
61
+  (let* ((cache-syms (mapcar (lambda (it)
62
+                               (list (gensym (symbol-name (first it)))
63
+                                     nil))
64
+                             bindings))
65
+         (cache-bindings (mapcar (lambda (sym cache-sym)
66
+                                   `(,(first sym) (or ,(first cache-sym)
67
+                                                      (setf ,(first cache-sym)
68
+                                                            ,(second sym)))))
69
+                                 bindings
70
+                                 cache-syms)))
71
+    `(let ,cache-syms
72
+       (declare (ignorable ,@(mapcar 'first cache-syms)))
73
+       (symbol-macrolet ,cache-bindings
74
+         ,@body))))
75
+
76
+(defmacro hash-table-dbind (bindings h-t &body body)
77
+  (alexandria:once-only (h-t)
78
+    `(let-lazy ,(loop for (name key . rest) in bindings
79
+                      collect `(,name (gethash ,key ,h-t ,@rest)))
80
+       ,@body)))
81
+
82
+(defmacro destructure-resource (resource &body body)
83
+  `(hash-table-dbind ((id "id")
84
+                      (type "type")
85
+                      (attributes "attributes" 'empty)
86
+                      (relationships "relationships" 'empty)
87
+                      (links "links" 'empty)
88
+                      (meta "meta" 'empty))
89
+       ,resource
90
+     ,@body))
91
+
92
+
93
+(defgeneric id-resolver-gf (store resource))
94
+(defmethod id-resolver-gf ((store jsonapi-datastore) (resource hash-table))
95
+  (destructure-resource resource
96
+    (get-object store type id)))
97
+(defmethod id-resolver-gf ((store jsonapi-datastore) (resource string))
98
+  (patmatch:let-pat* (((vector type id) (fwoar.string-utils:split "/" resource :count 1)))
99
+    (get-object store
100
+                type
101
+                id)))
102
+
103
+(defun id-resolver (store)
104
+  (lambda (resource)
105
+    (id-resolver-gf store resource)))
106
+
107
+(defmethod add-object :before ((store recording-datastore) (object hash-table))
108
+  (vector-push-extend (destructure-resource object
109
+                        (list :add-object id type attributes))
110
+                      (record store)))
111
+
112
+(defmethod stored-types ((store jsonapi-datastore))
113
+  (alexandria:hash-table-keys (database store)))
114
+
115
+
116
+(defmethod ingest-jsonapi ((store jsonapi-datastore) (object hash-table))
117
+  (hash-table-dbind ((data "data" 'empty)
118
+                     (errors "errors" 'empty)
119
+                     (meta "meta" 'empty)
120
+                     (jsonapi "jsonapi" 'empty)
121
+                     (links "links" 'empty)
122
+                     (included "included" 'empty))
123
+      object
124
+    (when (and (eql data 'empty)
125
+               (eql errors 'empty)
126
+               (eql meta 'empty))
127
+      (warn "one of data, errors or meta must be specified in a toplevel object"))
128
+    (when (not (or (eql data 'empty)
129
+                   (eql errors 'empty)))
130
+      (warn "only one of data or errors may be specified"))
131
+    (unless (when (eql data 'empty)
132
+              (not (eql included 'empty)))
133
+      (warn "included may only be specified if data is specified"))
134
+    (if (typep included 'sequence)
135
+        (add-object store included)
136
+        (unless (eql included 'empty)
137
+          (warn "included should be a sequence")
138
+          (add-object store included)))
139
+    (add-object store data)))
140
+
141
+(defun resource-identifier-p (resource)
142
+  (destructure-resource resource
143
+    (and resource
144
+         (eql attributes 'empty)
145
+         (eql relationships 'empty)
146
+         (eql links 'empty))))
147
+
148
+(defmacro with-json-output ((stream &key indent) &body body)
149
+  "Set up a JSON streaming encoder context on STREAM, then evaluate BODY."
150
+  `(let* ((yason::*json-output*
151
+            (make-instance 'yason::json-output-stream
152
+                           :output-stream ,stream
153
+                           :indent ,indent))
154
+          (,stream yason::*json-output*))
155
+     ,@body))
156
+
157
+(defmethod add-object ((store jsonapi-datastore) (object hash-table))
158
+  (destructure-resource object
159
+    (let* ((store-ht (database store))
160
+           (type-store (alexandria:ensure-gethash (normalize-type type)
161
+                                                  store-ht
162
+                                                  (make-hash-table :test 'equal)))
163
+           (old-value (gethash id type-store)))
164
+      (unless (eql relationships 'empty)
165
+        (loop for v being the hash-values of relationships
166
+              for data = (gethash "data" v)
167
+              when data do
168
+                (add-object store data)))
169
+      (if (resource-identifier-p old-value)
170
+          (setf (gethash id type-store) object)
171
+          (if (resource-identifier-p object)
172
+              old-value
173
+              (setf (gethash id type-store) object))))))
174
+
175
+(defmethod add-object ((store jsonapi-datastore) (objects sequence))
176
+  (map 'list
177
+       (lambda (it)
178
+         (add-object store it))
179
+       objects))
180
+
181
+(defmethod get-object ((store jsonapi-datastore) type id)
182
+  (let* ((store-ht (database store))
183
+         (type-store (alexandria:ensure-gethash (normalize-type type)
184
+                                                store-ht
185
+                                                (make-hash-table :test 'equal))))
186
+    (gethash id type-store)))
187
+
188
+(defun summarize-objects-of-type (store type map-fn filter-fn)
189
+  (serapeum:collecting
190
+    (serapeum:do-hash-table (k v (gethash type (database store)))
191
+      (declare (ignore k))
192
+      (when (funcall filter-fn v)
193
+        (collect (funcall map-fn v))))))
194
+
6 195
 (defun get-signin-page ()
7 196
   (let ((c-j (make-instance 'drakma:cookie-jar)))
8 197
     (values (plump:parse
... ...
@@ -12,8 +201,99 @@
12 201
 
13 202
 (defun extract-auth-token (doc)
14 203
   (lquery:$1 (inline doc)
15
-             "form input[type=hidden][name=authenticity_token]"
16
-             (attr "value")))
204
+    "form input[type=hidden][name=authenticity_token]"
205
+    (attr "value")))
206
+
207
+(defvar *csrf-token* nil)
208
+(defun request-json (url cj &rest r &key (method :get) (parameters ()))
209
+  (declare (ignore method parameters))
210
+  (let ((drakma:*text-content-types* (acons "application" "json"
211
+                                            drakma:*text-content-types*))
212
+        (yason:*parse-json-booleans-as-symbols* t)
213
+        #+(or)(yason:*parse-json-null-as-keyword* t)
214
+        (yason:*parse-json-arrays-as-vectors* t))
215
+    #+(or)identity
216
+    (yason:parse
217
+     (apply #'drakma:http-request url
218
+            :cookie-jar cj
219
+            :accept "application/json"
220
+            :additional-headers (when *csrf-token*
221
+                                  `(("X-CSRF-Token" . ,*csrf-token*)))
222
+            r))))
223
+
224
+(defun login-garagiste (auth-token cj)
225
+  (drakma:http-request "https://app.garagiste.com/users/sign_in"
226
+                       :cookie-jar cj
227
+                       :method :post
228
+                       :parameters `(("utf8" . ,(drakma:url-encode "✓" :utf-8))
229
+                                     ("authenticity_token" . ,auth-token)
230
+                                     ("user[email]" . ,(swank:ed-rpc
231
+                                                        'get-passwd :garagiste-email
232
+                                                        "Garagiste Email? "))
233
+                                     ("user[password]" . ,(swank:ed-rpc
234
+                                                           'get-passwd :garagiste
235
+                                                           "Garagiste Password? "))
236
+                                     ("user[remember_me]" . "0")
237
+                                     ("commit" . "Log in"))))
238
+
239
+
240
+(defun get-user-data ()
241
+  (multiple-value-bind (doc cookie-jar) (get-signin-page)
242
+    (let* ((auth-token (extract-auth-token doc))
243
+           (user-home (login-garagiste auth-token cookie-jar)))
244
+      (setf *csrf-token*
245
+            (lquery:$1 (initialize user-home)
246
+              "meta[name=csrf-token]"
247
+              (attr "content")))
248
+      (values (yason:parse
249
+               (lquery:$1 (initialize user-home)
250
+                 "div[data-react-class=Portal]"
251
+                 (attr "data-react-props")))
252
+              cookie-jar))))
253
+
254
+(defun user-id (ud)
255
+  (fw.lu:dive '("data" "customerId")
256
+              ud))
257
+
258
+(defun customer (user-id cookie-jar)
259
+  (request-json (format nil "https://app.garagiste.com/customers/~a" user-id)
260
+                cookie-jar))
261
+
262
+(defun requests (user-id cookie-jar)
263
+  (request-json (format nil "https://app.garagiste.com/customers/~a/requests/" user-id)
264
+                cookie-jar))
265
+
266
+(defun orders (user-id cookie-jar)
267
+  (request-json (format nil "https://app.garagiste.com/customers/~a/orders/" user-id)
268
+                cookie-jar))
269
+
270
+(defun shipments (user-id cookie-jar)
271
+  (request-json (format nil "https://app.garagiste.com/customers/~a/shipments/" user-id)
272
+                cookie-jar))
273
+
274
+
275
+(data-lens:defalias simplify-order
276
+  (data-lens:<>1 (lambda (it)
277
+                   (mapcan (lambda (key el)
278
+                             (list key el))
279
+                           '(:unit-price :count :total-price :offering-name)
280
+                           it))
281
+                 (data-lens:juxt (data-lens:key "unit_price")
282
+                                 (data-lens:key "count")
283
+                                 (data-lens:key "total_price")
284
+                                 (data-lens:key "offering_name"))))
285
+
286
+(data-lens:defalias summarize-orders
287
+  (data-lens:<>1 (data-lens:denest)
288
+                 (data-lens:transform-head
289
+                  (lambda (it)
290
+                    (list :id it)))
291
+                 (data-lens:juxt (data-lens:juxt (data-lens:<>1 'normalize-type
292
+                                                                (data-lens:key "type"))
293
+                                                 (data-lens:key "id"))
294
+                                 (data-lens:<>1 'simplify-order
295
+                                                (data-lens:key "attributes")))))
296
+
297
+(defclass test-store (jsonapi-datastore recording-datastore)
298
+  ())
17 299
 
18
-(defun login-garagiste (user pass auth-token)
19
-  )