# HG changeset patch # User Arne Babenhauserheide # Date 1704009004 -3600 # Sun Dec 31 08:50:04 2023 +0100 # Node ID fc41ef0b0d78033b7bfb3e44d9f15bcfdd3840a1 # Parent 4fa59042b2ee3e88266dc34a89a65d4fcedb7439 implement subscription handling diff --git a/wispwot/server.w b/wispwot/server.w --- a/wispwot/server.w +++ b/wispwot/server.w @@ -241,7 +241,7 @@ ;; We can safely tombstone the ID and empty the ;; trustlists. They are deleted in a deferred reindexing ;; (~garbage collecting) step. - + ;; We need to also check the ranks, because the ownid may ;; have no trust define indexes-without-any-trust @@ -429,7 +429,7 @@ wotcache-scores update-wotcache-from-wotstate! wotstate-with-cache ownid define otherindex : key->index otherkey - cond + cond : equal? ownkey otherkey . "100" ;; trust of own ID is always 100 {otherindex >= (vector-length scores)} @@ -580,17 +580,116 @@ define updated cons (car entry) : cons id : cdr entry list-set! subs index updated -define-syntax-rule : add-to-subscription! subs ownid id - begin +define-syntax-rule : ensure-ownid-entry-exists! subs ownid unless : assoc ownid subs set! subs alist-cons ownid '() subs +define-syntax-rule : add-to-subscription! subs ownid id + begin + ensure-ownid-entry-exists! subs ownid add-for-ownid! subs ownid id -define : get-subscriptions ownid +define : set-subscriptions-for-ownid-helper! subs ownid updated + define index + list-index : λ (x) : equal? ownid : first x + . subs + list-set! subs index updated +define-syntax-rule : set-subscriptions-for-ownid! subs ownid updated + begin + ensure-ownid-entry-exists! subs ownid + set-subscriptions-for-ownid-helper! subs ownid updated +define : get-own-subscriptions subs ownid + or : assoc-ref subs ownid + . '() +define : next-subscriptions wotstate ownid + . "Calculate the subscriptions to use in the next step." ;; TODO: update subscription-lists from wot and retrieve the ;; defined IDs. - or : assoc-ref subscriptions-updated ownid - . '() + + ;; Run a pruning check against those that did not update. Then + ;; this is driving the WoT again. + ;; limitfor indirect subscriptions per type + define max-indirect 25 + + ;; update the wotstate + define ranks + wotcache-ranks + update-wotcache-from-wotstate! wotstate ownid + define known-ids : wotstate-known-ids wotstate + define id-count : vector-length known-ids + define updated-rank1-ids '() + define updated-rank2-ids '() + define updated-rank3+-ids '() + define failed-rank2 '() + define failed-rank3+ '() + define id-to-index-map : make-id-to-index-map wotstate + define : get-updated-ids + or : assoc-ref subscriptions-updated ownid + . '() + define : get-rank id + and=> : hash-ref id-to-index-map id + cut ranks-ref ranks <> + ;; sort the updated IDs iby rank + let loop : : updated : get-updated-ids + unless : null? updated + let* + : id : first updated + rank : get-rank id + cond + {rank = 1} : set! updated-rank1-ids : cons id updated-rank1-ids + {rank = 2} : set! updated-rank2-ids : cons id updated-rank2-ids + {rank > 2} : set! updated-rank3+-ids : cons id updated-rank3+-ids + else #t + loop : cdr updated + ;; clear updated IDs + set-subscriptions-for-ownid! subscriptions-updated ownid '() + ;; reorder rank1 subscriptions + let : : rank1-subs : get-own-subscriptions rank1-subscriptions ownid + set-subscriptions-for-ownid! rank1-subscriptions ownid + delete-duplicates + append updated-rank1-ids rank1-subscriptions + let : : rank2-subs-recent : get-own-subscriptions rank2-subscriptions-most-recent ownid + define rank2-subs + delete-duplicates + append updated-rank2-ids rank2-subscriptions-most-recent + set! failed-rank2 + drop rank2-subs : min max-indirect : length rank2-subscriptions-most-recent + set-subscriptions-for-ownid! rank2-subscriptions-most-recent ownid + take rank2-subs : min max-indirect : length rank2-subs + ;; TDOD: add pruning + let : : rank3+-subs-recent : get-own-subscriptions rank3+-subscriptions-most-recent ownid + define rank3+-subs + delete-duplicates + append updated-rank3+-ids rank3+-subscriptions-most-recent + set! failed-rank3+ + drop rank3+-subs : min max-indirect : length rank3+-subscriptions-most-recent + set-subscriptions-for-ownid! rank3+-subscriptions-most-recent ownid + take rank3+-subs : min max-indirect : length rank3+-subs + ;; choose at random TODO: make less primitive + let loop : (rank2-ids '()) (rank3+-ids '()) (tries {2 * max-indirect}) + define index : random id-count + define id : vector-ref known-ids index + define rank : ranks-ref ranks index + cond + : zero? tries + set-subscriptions-for-ownid! rank2-subscriptions-random ownid + delete-duplicates rank2-ids + set-subscriptions-for-ownid! rank3+-subscriptions-random ownid + delete-duplicates rank3+-ids + {rank = 2} + loop : cons id rank2-ids + . rank3+-ids {tries - 1} + {rank > 2} + loop rank2-ids + cons id rank3+-ids + . {tries - 1} + else + loop rank2-ids rank3+-ids {tries - 1} + append + . rank1-subscriptions + . rank2-subscriptions-most-recent + . rank2-subscriptions-random + . rank3+-subscriptions-most-recent + . rank3+-subscriptions-random ;; alist of hash-tables (one per ownid) with ids as key and alists of metadata as value @@ -613,7 +712,7 @@ define-handler 'PUT "/subscription-updated/ownkey/" : put-subscription-updated-handler request body wotstate . "Endpoint: /subscription-updated/ownkey/OWN/otherid/ID - + Add metadata needed to check the ID as form-data. " define path : split-and-decode-uri-path : uri-path : request-uri request @@ -638,17 +737,17 @@ build-response . #:headers `((content-type . (text/plain))) . "" - + define-handler 'GET "/subscriptions/" : get-subscriptions-handler request body wotstate . "Endpoint: /subscriptions/ownid - + Returns a list of IDs to check for updates, separated by empty lines." define path-raw : uri-path : request-uri request define path : string-join (split-and-decode-uri-path path-raw) "/" define ownid : string-drop path : string-length "subscriptions/" ;; TODO: get subscriptions for ownid - define subscriptions : get-subscriptions ownid + define subscriptions : next-subscriptions wotstate ownid define : metadata-strings ownid id define metadata : get-metadata ownid id map : lambda (cel) : string-join (list (first cel) (second cel)) ": " diff --git a/wispwot/wispwot.w b/wispwot/wispwot.w --- a/wispwot/wispwot.w +++ b/wispwot/wispwot.w @@ -27,7 +27,7 @@ define-module : wispwot wispwot - . #:export : wispwot-read-trustfile wispwot-get-score read-known-identities make-wotstate wotstate-known-ids wotstate-trustlists trustlists-empty wotstate-ranks wotstate-scores read-all-trust calculate-ranks calculate-scores import-trust-csv get-trust-edge add-trust-edge ids-max ids->list id->key key->index index->identity write-store update-ranks-and-scores-from-trust-edge! update-ranks-and-scores-from-trust-identity-edges! ranks->list list->ranks ranks-ref set-wotstate-known-ids! set-wotstate-trustlists! + . #:export : wispwot-read-trustfile wispwot-get-score read-known-identities make-wotstate wotstate-known-ids wotstate-trustlists trustlists-empty wotstate-ranks wotstate-scores read-all-trust calculate-ranks calculate-scores import-trust-csv get-trust-edge add-trust-edge ids-max ids->list id->key key->index index->identity write-store update-ranks-and-scores-from-trust-edge! update-ranks-and-scores-from-trust-identity-edges! ranks->list list->ranks ranks-ref set-wotstate-known-ids! set-wotstate-trustlists! make-id-to-index-map import : wispwot doctests srfi srfi-1 ; lists