(drak)
2016-02-05: crawn: only redownload if #:redownload #t crawn: only redownload if #:redownload #t
diff --git a/crawl-wot.scm b/crawl-wot.scm --- a/crawl-wot.scm +++ b/crawl-wot.scm @@ -88,21 +88,31 @@ (let ((u (if (string-prefix? "freenet:" uri) (substring uri 8) uri))) + (format #t "Download to file ~A\n" filename) (if (string-prefix? "USK@" u) (let ((data (get (furl-uri u)))) (if (string? data) (let ((port (open-output-file filename))) (put-string port data) (close-port port)) - (error (format #t "tried to save in file ~A" filename)))) - (error (format #t "tried to save in file ~A" filename))))) + (error (format #t "tried to save in file ~A\n" filename)))) + (error (format #t "tried to save in file ~A\n" filename))))) -(define (crawl-wot seed-id) +(define* (crawl-wot seed-id #:key (redownload #f)) (let ((known '())) (let crawl ((seed seed-id)) ;; save the data (if (catch 'misc-error - (lambda () (dump-wot-id seed (wot-uri-filename seed)) #f) + (lambda () (let* ((filename (wot-uri-filename seed)) + (dump (lambda () (dump-wot-id seed filename)))) + (if (and (not redownload) (file-exists? filename)) + (let* ((s (stat filename)) + (size (stat:size s))) + (if (= size 0) + (dump) + (format #t "Use local copy of file ~A (redownload ~A).\n" filename redownload))) + (dump)) + #f)) (lambda (key . args) #t)) known ;; snarf all uris