(drak)
2016-02-16: add status for dl by date hint add status for dl by date hint
diff --git a/crawl-wot.scm b/crawl-wot.scm
--- a/crawl-wot.scm
+++ b/crawl-wot.scm
@@ -134,10 +134,10 @@
(not (pair? u)) ; TODO: this is a hack. I do not know why u can be the full sxml. Seems to happen with IDs who do not have any trust set.
(not (member (wot-uri-key u) known)))) u)))
(when (not (null? new))
- (display 'new:)
+ (display "new: ")
(write (car new))(newline))
(when (not (null? known))
- (display 'known:)
+ (display "known: ")
(write (car known))(newline)(write (length known))(newline))
(set! known (lset-union equal?
(list-ec (: u new) (wot-uri-key u))
@@ -175,6 +175,7 @@
(filename (string-append date "/" (wot-uri-key uri) "-" version)))
(when (not (file-exists? date))
(mkdir date))
+ (format #t "download to: ~A | for week ~A\n" filename week)
(let ((data (get url)))
(when (string? data)
(let ((port (open-output-file filename)))
@@ -195,23 +196,23 @@
;; Approach: First check whether the ID has a date hint for each year. Then check each weak in the matching years.
;; download the versions into directories ordered as YEAR-month-day/SSK@...-WebOfTrust-version
(let ((years (iota 10 2016 -1)) ; last 10 years
- (weeks (iota 2 1))) ; 1-52
+ (weeks (iota 52 1))) ; 1-52
(delete #f ;; only return the filenames of successful downloads
(par-map (lambda (year)
(let* ((yearuri (datehint-for-key (wot-uri-key uri) year))
- (hint (get (furl-uri yearuri)))
- (hint-alist (parse-datehint hint))
- (date (assoc-ref hint-alist 'date))
- (month (string->number (list-ref (string-split date #\-) 2)))
- (min-week (* (- month 1) 4))) ; avoid trying to download weeks which cannot be available.
+ (hint (get (furl-uri yearuri))))
(if (not (string? hint))
#f
- (delete #f ;; only return the filenames of successful downloads
- (n-par-map 52 (lambda (week)
- (if (< week min-week) ; avoid weeks earlier than the date in the yearly date hint
- #f
- (download-by-weekly-date-hint uri year week)))
- weeks)))))
+ (let* ((hint-alist (parse-datehint hint))
+ (date (assoc-ref hint-alist 'date))
+ (month (string->number (list-ref (string-split date #\-) 2)))
+ (min-week (* (- month 1) 4))) ; avoid trying to download weeks which cannot be available.
+ (delete #f ;; only return the filenames of successful downloads
+ (n-par-map 52 (lambda (week)
+ (if (< week min-week) ; avoid weeks earlier than the date in the yearly date hint
+ #f
+ (download-by-weekly-date-hint uri year week)))
+ weeks))))))
years))))
(define (main args)
@@ -222,5 +223,5 @@
seed-id
(string-append "USK" (string-drop seed-id 3) "/WebOfTrust/0"))))
;; (write (download-by-date-hint seed))
- (par-map (lambda (x) (map download-by-date-hint x))
+ (par-map download-by-date-hint
(crawl-wot seed)))))