Overview
Comment: | Added support for allowing the user to completely control how downloads are performed as well as configure the default method |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: | 3864351ad443a845f4479be044e9bf5eafb0dc44 |
User & Date: | rkeene on 2016-07-10 19:26:13 |
Other Links: | manifest | tags |
Context
2017-02-06
| ||
15:57 | Added support for a Tcl "exec" target to be compiled in check-in: 3d73fc5750 user: rkeene tags: trunk | |
2017-01-19
| ||
17:49 | Started work on completely supporting multiple hashing algorithms check-in: 2460a1ddab user: rkeene tags: complete-multihash | |
2016-07-10
| ||
19:26 | Added support for allowing the user to completely control how downloads are performed as well as configure the default method check-in: 3864351ad4 user: rkeene tags: trunk | |
19:05 | Fixed typo in debug message check-in: 936a791a4a user: rkeene tags: trunk | |
Changes
Modified appfsd.tcl from [865f14848a] to [985d2df621].
27 27 package require sha1 28 28 package require appfsd 29 29 package require platform 30 30 package require pki 31 31 32 32 # Functions specifically meant for users to replace as a part of configuration 33 33 namespace eval ::appfs::user { 34 + variable download_method "tcl" 35 + 34 36 # User-replacable function to convert a hostname/hash/method to an URL 35 37 proc construct_url {hostname hash method} { 36 38 return "http://$hostname/appfs/$method/$hash" 37 39 } 38 40 39 41 # User-replaceable function get the home directory of the current user 40 42 proc get_homedir {} { ................................................................................ 49 51 50 52 if {[info exists ::appfs::user::add_perms($sha1)]} { 51 53 append perms $::appfs::user::add_perms($sha1) 52 54 } 53 55 54 56 return $perms 55 57 } 58 + 59 + # User-replacable function to fetch a remote file 60 + proc download_file {url {outputChannel ""}} { 61 + switch -- $::appfs::user::download_method { 62 + "curl" { 63 + if {$outputChannel eq ""} { 64 + return [exec curl -sS -L -- $url] 65 + } else { 66 + exec curl -sS -L -- $url >@ $outputChannel 67 + 68 + return "" 69 + } 70 + } 71 + "tcl" { 72 + catch { 73 + if {$outputChannel eq ""} { 74 + set token [http::geturl $url] 75 + set retval [http::data $token] 76 + } else { 77 + set token [http::geturl $url -binary true -channel $outputChannel] 78 + set retval "" 79 + } 80 + } err 81 + 82 + if {![info exists token]} { 83 + return -code error "Unable to download \"$url\": $err" 84 + } 85 + 86 + set tokenCode [http::ncode $token] 87 + 88 + http::cleanup $token 89 + 90 + if {$tokenCode != "200"} { 91 + return -code error "Unable to download \"$url\": Site did not return a 200 (returned $tokenCode)" 92 + } 93 + 94 + if {![info exists retval]} { 95 + return -code error "Unable to download \"$url\": Site did not return proper data: $err" 96 + } 97 + 98 + return $retval 99 + } 100 + 101 + } 102 + 103 + return -code error "Unable to download" 104 + } 56 105 } 57 106 58 107 namespace eval ::appfs { 59 108 variable cachedir "/tmp/appfs-cache" 60 109 variable ttl 3600 61 110 variable nttl 60 62 111 variable trusted_cas [list] ................................................................................ 67 116 append retval "[string range $hash [expr {$idx * 2}] [expr {($idx * 2) + 1}]]/" 68 117 } 69 118 append retval "[string range $hash [expr {$idx * 2}] end]" 70 119 71 120 return $retval 72 121 } 73 122 74 - proc _cachefile {url key {keyIsHash 1}} { 123 + proc _cachefile {url key method {keyIsHash 1}} { 124 + if {$keyIsHash && $method != "sha1"} { 125 + return -code error "Only SHA1 hashing method is supported" 126 + } 127 + 75 128 set filekey $key 76 129 if {$keyIsHash} { 77 130 set filekey [_hash_sep $filekey] 78 131 } 79 132 80 133 set file [file join $::appfs::cachedir $filekey] 81 134 ................................................................................ 87 140 88 141 set tmpfile "${file}.[expr {rand()}][clock clicks]" 89 142 90 143 set fd [open $tmpfile "w"] 91 144 fconfigure $fd -translation binary 92 145 93 146 catch { 94 - set token [::http::geturl $url -channel $fd -binary true] 95 - } 96 - 97 - if {[info exists token]} { 98 - set ncode [::http::ncode $token] 99 - ::http::reset $token 100 - } else { 101 - set ncode "900" 147 + ::appfs::user::download_file $url $fd 102 148 } 103 149 104 150 close $fd 105 151 106 152 if {$keyIsHash} { 107 153 set hash [string tolower [sha1::sha1 -hex -file $tmpfile]] 108 154 } else { 109 155 set hash $key 110 156 } 111 157 112 - if {$ncode == "200" && $hash == $key} { 158 + if {$hash == $key} { 113 159 file rename -force -- $tmpfile $file 114 160 } else { 115 161 file delete -force -- $tmpfile 116 162 } 117 163 118 164 return $file 119 165 } ................................................................................ 274 320 db eval {CREATE INDEX IF NOT EXISTS sites_index ON sites (hostname);} 275 321 db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, sha1, package, version, os, cpuArch);} 276 322 db eval {CREATE INDEX IF NOT EXISTS files_index ON files (package_sha1, file_name, file_directory);} 277 323 } 278 324 279 325 proc download {hostname hash {method sha1}} { 280 326 set url [::appfs::user::construct_url $hostname $hash $method] 281 - set file [_cachefile $url $hash] 327 + set file [_cachefile $url $hash $method] 282 328 283 329 if {![file exists $file]} { 284 330 return -code error "Unable to fetch (file does not exist: $file)" 285 331 } 286 332 287 333 return $file 288 334 } ................................................................................ 306 352 if {$now < ($lastUpdate + $ttl)} { 307 353 return COMPLETE 308 354 } 309 355 310 356 set url "http://$hostname/appfs/index" 311 357 312 358 catch { 313 - set token [::http::geturl $url] 314 - if {[::http::ncode $token] == "200"} { 315 - set indexhash_data [::http::data $token] 316 - } 317 - ::http::reset $token 318 - ::http::cleanup $token 359 + set indexhash_data [::appfs::user::download_file $url] 319 360 } 320 361 321 362 # Note that we attempted to fetch this index and do not try 322 363 # again for a while 323 364 db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::nttl);} 324 365 325 366 if {![info exists indexhash_data]} {