Browse code
feat: improve garagiste puller
Ed L authored on 31/12/2019 05:29:29
Showing 1 changed files
Showing 1 changed files
... | ... |
@@ -3,6 +3,195 @@ |
3 | 3 |
(:export )) |
4 | 4 |
(in-package :garagiste-app) |
5 | 5 |
|
6 |
+(defclass recording-datastore () |
|
7 |
+ ((%record :accessor record |
|
8 |
+ :initform (make-array 10 |
|
9 |
+ :adjustable t |
|
10 |
+ :fill-pointer 0)))) |
|
11 |
+(defclass jsonapi-datastore () |
|
12 |
+ ((%database :reader database |
|
13 |
+ :initform (make-hash-table)))) |
|
14 |
+(defgeneric add-object (store object)) |
|
15 |
+(defgeneric get-object (store type id)) |
|
16 |
+(defgeneric ingest-jsonapi (store object)) |
|
17 |
+(defgeneric stored-types (store)) |
|
18 |
+ |
|
19 |
+(defun object (store type id) |
|
20 |
+ (get-object store type id)) |
|
21 |
+ |
|
22 |
+(defun (setf object) (new-value store type id) |
|
23 |
+ (let ((tmp (alexandria:copy-hash-table new-value))) |
|
24 |
+ (setf (gethash "id" tmp) id |
|
25 |
+ (gethash "type" tmp) type) |
|
26 |
+ (add-object store tmp))) |
|
27 |
+ |
|
28 |
+(defclass batch () |
|
29 |
+ ((%id :accessor id) |
|
30 |
+ (%shipments :accessor shipments-a))) |
|
31 |
+(defmethod add-object ((store jsonapi-datastore) (object batch)) |
|
32 |
+ (setf (object :batches (id object)) |
|
33 |
+ (alexandria:alist-hash-table |
|
34 |
+ `(("relationships" . ,(mapcar (lambda (it) |
|
35 |
+ (alexandria:alist-hash-table |
|
36 |
+ `(("type" . "shipments") |
|
37 |
+ ("id" . ,it)))) |
|
38 |
+ (shipments-a object))))))) |
|
39 |
+ |
|
40 |
+(defun normalize-type (type) |
|
41 |
+ (declare (optimize debug)) |
|
42 |
+ (let ((normalize-char (lambda (ch) |
|
43 |
+ (case ch |
|
44 |
+ (#\_ #\-) |
|
45 |
+ (t (if (lower-case-p ch) |
|
46 |
+ (char-upcase ch) |
|
47 |
+ (char-downcase ch))))))) |
|
48 |
+ (nth-value |
|
49 |
+ 0 (etypecase type |
|
50 |
+ (simple-string (intern (map 'simple-string |
|
51 |
+ normalize-char |
|
52 |
+ type) |
|
53 |
+ :keyword)) |
|
54 |
+ (string (intern (map 'simple-string |
|
55 |
+ normalize-char |
|
56 |
+ type) |
|
57 |
+ :keyword)) |
|
58 |
+ (keyword type))))) |
|
59 |
+ |
|
60 |
+(defmacro let-lazy ((&rest bindings) &body body) |
|
61 |
+ (let* ((cache-syms (mapcar (lambda (it) |
|
62 |
+ (list (gensym (symbol-name (first it))) |
|
63 |
+ nil)) |
|
64 |
+ bindings)) |
|
65 |
+ (cache-bindings (mapcar (lambda (sym cache-sym) |
|
66 |
+ `(,(first sym) (or ,(first cache-sym) |
|
67 |
+ (setf ,(first cache-sym) |
|
68 |
+ ,(second sym))))) |
|
69 |
+ bindings |
|
70 |
+ cache-syms))) |
|
71 |
+ `(let ,cache-syms |
|
72 |
+ (declare (ignorable ,@(mapcar 'first cache-syms))) |
|
73 |
+ (symbol-macrolet ,cache-bindings |
|
74 |
+ ,@body)))) |
|
75 |
+ |
|
76 |
+(defmacro hash-table-dbind (bindings h-t &body body) |
|
77 |
+ (alexandria:once-only (h-t) |
|
78 |
+ `(let-lazy ,(loop for (name key . rest) in bindings |
|
79 |
+ collect `(,name (gethash ,key ,h-t ,@rest))) |
|
80 |
+ ,@body))) |
|
81 |
+ |
|
82 |
+(defmacro destructure-resource (resource &body body) |
|
83 |
+ `(hash-table-dbind ((id "id") |
|
84 |
+ (type "type") |
|
85 |
+ (attributes "attributes" 'empty) |
|
86 |
+ (relationships "relationships" 'empty) |
|
87 |
+ (links "links" 'empty) |
|
88 |
+ (meta "meta" 'empty)) |
|
89 |
+ ,resource |
|
90 |
+ ,@body)) |
|
91 |
+ |
|
92 |
+ |
|
93 |
+(defgeneric id-resolver-gf (store resource)) |
|
94 |
+(defmethod id-resolver-gf ((store jsonapi-datastore) (resource hash-table)) |
|
95 |
+ (destructure-resource resource |
|
96 |
+ (get-object store type id))) |
|
97 |
+(defmethod id-resolver-gf ((store jsonapi-datastore) (resource string)) |
|
98 |
+ (patmatch:let-pat* (((vector type id) (fwoar.string-utils:split "/" resource :count 1))) |
|
99 |
+ (get-object store |
|
100 |
+ type |
|
101 |
+ id))) |
|
102 |
+ |
|
103 |
+(defun id-resolver (store) |
|
104 |
+ (lambda (resource) |
|
105 |
+ (id-resolver-gf store resource))) |
|
106 |
+ |
|
107 |
+(defmethod add-object :before ((store recording-datastore) (object hash-table)) |
|
108 |
+ (vector-push-extend (destructure-resource object |
|
109 |
+ (list :add-object id type attributes)) |
|
110 |
+ (record store))) |
|
111 |
+ |
|
112 |
+(defmethod stored-types ((store jsonapi-datastore)) |
|
113 |
+ (alexandria:hash-table-keys (database store))) |
|
114 |
+ |
|
115 |
+ |
|
116 |
+(defmethod ingest-jsonapi ((store jsonapi-datastore) (object hash-table)) |
|
117 |
+ (hash-table-dbind ((data "data" 'empty) |
|
118 |
+ (errors "errors" 'empty) |
|
119 |
+ (meta "meta" 'empty) |
|
120 |
+ (jsonapi "jsonapi" 'empty) |
|
121 |
+ (links "links" 'empty) |
|
122 |
+ (included "included" 'empty)) |
|
123 |
+ object |
|
124 |
+ (when (and (eql data 'empty) |
|
125 |
+ (eql errors 'empty) |
|
126 |
+ (eql meta 'empty)) |
|
127 |
+ (warn "one of data, errors or meta must be specified in a toplevel object")) |
|
128 |
+ (when (not (or (eql data 'empty) |
|
129 |
+ (eql errors 'empty))) |
|
130 |
+ (warn "only one of data or errors may be specified")) |
|
131 |
+ (unless (when (eql data 'empty) |
|
132 |
+ (not (eql included 'empty))) |
|
133 |
+ (warn "included may only be specified if data is specified")) |
|
134 |
+ (if (typep included 'sequence) |
|
135 |
+ (add-object store included) |
|
136 |
+ (unless (eql included 'empty) |
|
137 |
+ (warn "included should be a sequence") |
|
138 |
+ (add-object store included))) |
|
139 |
+ (add-object store data))) |
|
140 |
+ |
|
141 |
+(defun resource-identifier-p (resource) |
|
142 |
+ (destructure-resource resource |
|
143 |
+ (and resource |
|
144 |
+ (eql attributes 'empty) |
|
145 |
+ (eql relationships 'empty) |
|
146 |
+ (eql links 'empty)))) |
|
147 |
+ |
|
148 |
+(defmacro with-json-output ((stream &key indent) &body body) |
|
149 |
+ "Set up a JSON streaming encoder context on STREAM, then evaluate BODY." |
|
150 |
+ `(let* ((yason::*json-output* |
|
151 |
+ (make-instance 'yason::json-output-stream |
|
152 |
+ :output-stream ,stream |
|
153 |
+ :indent ,indent)) |
|
154 |
+ (,stream yason::*json-output*)) |
|
155 |
+ ,@body)) |
|
156 |
+ |
|
157 |
+(defmethod add-object ((store jsonapi-datastore) (object hash-table)) |
|
158 |
+ (destructure-resource object |
|
159 |
+ (let* ((store-ht (database store)) |
|
160 |
+ (type-store (alexandria:ensure-gethash (normalize-type type) |
|
161 |
+ store-ht |
|
162 |
+ (make-hash-table :test 'equal))) |
|
163 |
+ (old-value (gethash id type-store))) |
|
164 |
+ (unless (eql relationships 'empty) |
|
165 |
+ (loop for v being the hash-values of relationships |
|
166 |
+ for data = (gethash "data" v) |
|
167 |
+ when data do |
|
168 |
+ (add-object store data))) |
|
169 |
+ (if (resource-identifier-p old-value) |
|
170 |
+ (setf (gethash id type-store) object) |
|
171 |
+ (if (resource-identifier-p object) |
|
172 |
+ old-value |
|
173 |
+ (setf (gethash id type-store) object)))))) |
|
174 |
+ |
|
175 |
+(defmethod add-object ((store jsonapi-datastore) (objects sequence)) |
|
176 |
+ (map 'list |
|
177 |
+ (lambda (it) |
|
178 |
+ (add-object store it)) |
|
179 |
+ objects)) |
|
180 |
+ |
|
181 |
+(defmethod get-object ((store jsonapi-datastore) type id) |
|
182 |
+ (let* ((store-ht (database store)) |
|
183 |
+ (type-store (alexandria:ensure-gethash (normalize-type type) |
|
184 |
+ store-ht |
|
185 |
+ (make-hash-table :test 'equal)))) |
|
186 |
+ (gethash id type-store))) |
|
187 |
+ |
|
188 |
+(defun summarize-objects-of-type (store type map-fn filter-fn) |
|
189 |
+ (serapeum:collecting |
|
190 |
+ (serapeum:do-hash-table (k v (gethash type (database store))) |
|
191 |
+ (declare (ignore k)) |
|
192 |
+ (when (funcall filter-fn v) |
|
193 |
+ (collect (funcall map-fn v)))))) |
|
194 |
+ |
|
6 | 195 |
(defun get-signin-page () |
7 | 196 |
(let ((c-j (make-instance 'drakma:cookie-jar))) |
8 | 197 |
(values (plump:parse |
... | ... |
@@ -12,8 +201,99 @@ |
12 | 201 |
|
13 | 202 |
(defun extract-auth-token (doc) |
14 | 203 |
(lquery:$1 (inline doc) |
15 |
- "form input[type=hidden][name=authenticity_token]" |
|
16 |
- (attr "value"))) |
|
204 |
+ "form input[type=hidden][name=authenticity_token]" |
|
205 |
+ (attr "value"))) |
|
206 |
+ |
|
207 |
+(defvar *csrf-token* nil) |
|
208 |
+(defun request-json (url cj &rest r &key (method :get) (parameters ())) |
|
209 |
+ (declare (ignore method parameters)) |
|
210 |
+ (let ((drakma:*text-content-types* (acons "application" "json" |
|
211 |
+ drakma:*text-content-types*)) |
|
212 |
+ (yason:*parse-json-booleans-as-symbols* t) |
|
213 |
+ #+(or)(yason:*parse-json-null-as-keyword* t) |
|
214 |
+ (yason:*parse-json-arrays-as-vectors* t)) |
|
215 |
+ #+(or)identity |
|
216 |
+ (yason:parse |
|
217 |
+ (apply #'drakma:http-request url |
|
218 |
+ :cookie-jar cj |
|
219 |
+ :accept "application/json" |
|
220 |
+ :additional-headers (when *csrf-token* |
|
221 |
+ `(("X-CSRF-Token" . ,*csrf-token*))) |
|
222 |
+ r)))) |
|
223 |
+ |
|
224 |
+(defun login-garagiste (auth-token cj) |
|
225 |
+ (drakma:http-request "https://app.garagiste.com/users/sign_in" |
|
226 |
+ :cookie-jar cj |
|
227 |
+ :method :post |
|
228 |
+ :parameters `(("utf8" . ,(drakma:url-encode "✓" :utf-8)) |
|
229 |
+ ("authenticity_token" . ,auth-token) |
|
230 |
+ ("user[email]" . ,(swank:ed-rpc |
|
231 |
+ 'get-passwd :garagiste-email |
|
232 |
+ "Garagiste Email? ")) |
|
233 |
+ ("user[password]" . ,(swank:ed-rpc |
|
234 |
+ 'get-passwd :garagiste |
|
235 |
+ "Garagiste Password? ")) |
|
236 |
+ ("user[remember_me]" . "0") |
|
237 |
+ ("commit" . "Log in")))) |
|
238 |
+ |
|
239 |
+ |
|
240 |
+(defun get-user-data () |
|
241 |
+ (multiple-value-bind (doc cookie-jar) (get-signin-page) |
|
242 |
+ (let* ((auth-token (extract-auth-token doc)) |
|
243 |
+ (user-home (login-garagiste auth-token cookie-jar))) |
|
244 |
+ (setf *csrf-token* |
|
245 |
+ (lquery:$1 (initialize user-home) |
|
246 |
+ "meta[name=csrf-token]" |
|
247 |
+ (attr "content"))) |
|
248 |
+ (values (yason:parse |
|
249 |
+ (lquery:$1 (initialize user-home) |
|
250 |
+ "div[data-react-class=Portal]" |
|
251 |
+ (attr "data-react-props"))) |
|
252 |
+ cookie-jar)))) |
|
253 |
+ |
|
254 |
+(defun user-id (ud) |
|
255 |
+ (fw.lu:dive '("data" "customerId") |
|
256 |
+ ud)) |
|
257 |
+ |
|
258 |
+(defun customer (user-id cookie-jar) |
|
259 |
+ (request-json (format nil "https://app.garagiste.com/customers/~a" user-id) |
|
260 |
+ cookie-jar)) |
|
261 |
+ |
|
262 |
+(defun requests (user-id cookie-jar) |
|
263 |
+ (request-json (format nil "https://app.garagiste.com/customers/~a/requests/" user-id) |
|
264 |
+ cookie-jar)) |
|
265 |
+ |
|
266 |
+(defun orders (user-id cookie-jar) |
|
267 |
+ (request-json (format nil "https://app.garagiste.com/customers/~a/orders/" user-id) |
|
268 |
+ cookie-jar)) |
|
269 |
+ |
|
270 |
+(defun shipments (user-id cookie-jar) |
|
271 |
+ (request-json (format nil "https://app.garagiste.com/customers/~a/shipments/" user-id) |
|
272 |
+ cookie-jar)) |
|
273 |
+ |
|
274 |
+ |
|
275 |
+(data-lens:defalias simplify-order |
|
276 |
+ (data-lens:<>1 (lambda (it) |
|
277 |
+ (mapcan (lambda (key el) |
|
278 |
+ (list key el)) |
|
279 |
+ '(:unit-price :count :total-price :offering-name) |
|
280 |
+ it)) |
|
281 |
+ (data-lens:juxt (data-lens:key "unit_price") |
|
282 |
+ (data-lens:key "count") |
|
283 |
+ (data-lens:key "total_price") |
|
284 |
+ (data-lens:key "offering_name")))) |
|
285 |
+ |
|
286 |
+(data-lens:defalias summarize-orders |
|
287 |
+ (data-lens:<>1 (data-lens:denest) |
|
288 |
+ (data-lens:transform-head |
|
289 |
+ (lambda (it) |
|
290 |
+ (list :id it))) |
|
291 |
+ (data-lens:juxt (data-lens:juxt (data-lens:<>1 'normalize-type |
|
292 |
+ (data-lens:key "type")) |
|
293 |
+ (data-lens:key "id")) |
|
294 |
+ (data-lens:<>1 'simplify-order |
|
295 |
+ (data-lens:key "attributes"))))) |
|
296 |
+ |
|
297 |
+(defclass test-store (jsonapi-datastore recording-datastore) |
|
298 |
+ ()) |
|
17 | 299 |
|
18 |
-(defun login-garagiste (user pass auth-token) |
|
19 |
- ) |