Check-in [7ed2f89c7d]
Overview
Comment:Updated to remove packages from cache that have been removed from the server
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:7ed2f89c7de1997ccb26a7112957072e39df2cb2
User & Date: rkeene on 2014-09-10 08:47:06
Other Links: manifest | tags
Context
2014-09-10
09:14
Minor update check-in: aa1acadcb1 user: rkeene tags: trunk
08:47
Updated to remove packages from cache that have been removed from the server check-in: 7ed2f89c7d user: rkeene tags: trunk
08:23
Updated to expire cache of server index periodically check-in: 977195a680 user: rkeene tags: trunk
Changes

Modified appfsd.tcl from [7312652049] to [ccdd0bdb23].

    60     60   				file delete -force -- $tmpfile
    61     61   			}
    62     62   		}
    63     63   
    64     64   		return $file
    65     65   	}
    66     66   
           67  +
           68  +	proc _isHash {value} {
           69  +		set value [string tolower $value]
           70  +
           71  +		if {[string length $value] != 40} {
           72  +			return false
           73  +		}
           74  +
           75  +		if {![regexp {^[0-9a-f]*$} $value]} {
           76  +			return false
           77  +		}
           78  +
           79  +		return true
           80  +	}
           81  +
    67     82   	proc _db {args} {
    68     83   		return [uplevel 1 [list ::appfs::db {*}$args]]
    69     84   	}
    70     85   
    71     86   	proc init {} {
    72     87   		if {[info exists ::appfs::init_called]} {
    73     88   			return
................................................................................
    77     92   
    78     93   		if {![info exists ::appfs::db]} {
    79     94   			file mkdir $::appfs::cachedir
    80     95   
    81     96   			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]
    82     97   		}
    83     98   
    84         -		_db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate);}
           99  +		_db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate, ttl);}
    85    100   		_db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest);}
    86    101   		_db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory);}
    87    102   	}
    88    103   
    89    104   	proc download {hostname hash {method sha1}} {
    90    105   		set url "http://$hostname/appfs/$method/$hash"
    91    106   		set file [_cachefile $url $hash]
................................................................................
    96    111   
    97    112   		return $file
    98    113   	}
    99    114   
   100    115   	proc getindex {hostname} {
   101    116   		set now [clock seconds]
   102    117   
   103         -		set lastUpdates [_db eval {SELECT lastUpdate FROM sites WHERE hostname = $hostname LIMIT 1;}]
          118  +		set lastUpdates [_db eval {SELECT lastUpdate, ttl FROM sites WHERE hostname = $hostname LIMIT 1;}]
   104    119   		if {[llength $lastUpdates] == 0} {
   105    120   			set lastUpdate 0
          121  +			set ttl 0
   106    122   		} else {
   107    123   			set lastUpdate [lindex $lastUpdates 0]
          124  +			set ttl [lindex $lastUpdates 1]
   108    125   		}
   109    126   
   110         -		if {$now < ($lastUpdate + $::appfs::ttl)} {
          127  +		if {$now < ($lastUpdate + $ttl)} {
   111    128   			return COMPLETE
   112    129   		}
   113    130   
   114    131   		if {[string match "*\[/~\]*" $hostname]} {
   115    132   			return -code error "Invalid hostname"
   116    133   		}
   117    134   
................................................................................
   123    140   				set indexhash_data [::http::data $token]
   124    141   			}
   125    142   			::http::reset $token
   126    143   			$token cleanup
   127    144   		}
   128    145   
   129    146   		if {![info exists indexhash_data]} {
          147  +			# Cache this result for 60 seconds
          148  +			_db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, 60);}
          149  +
   130    150   			return -code error "Unable to fetch $url"
   131    151   		}
   132    152   
   133    153   		set indexhash [lindex [split $indexhash_data ","] 0]
          154  +
          155  +		if {![_isHash $indexhash]} {
          156  +			return -code error "Invalid hash: $indexhash"
          157  +		}
   134    158   
   135    159   		set file [download $hostname $indexhash]
   136    160   		set fd [open $file]
   137    161   		set data [read $fd]
   138    162   		close $fd
   139    163   
   140         -		array set packages [list]
          164  +		set curr_packages [list]
   141    165   		foreach line [split $data "\n"] {
   142    166   			set line [string trim $line]
   143    167   
   144    168   			if {[string match "*/*" $line]} {
   145    169   				continue
   146    170   			}
   147    171   
................................................................................
   156    180   			set pkgInfo(version)  [lindex $work 1]
   157    181   			set pkgInfo(os)       [lindex $work 2]
   158    182   			set pkgInfo(cpuArch)  [lindex $work 3]
   159    183   			set pkgInfo(hash)     [string tolower [lindex $work 4]]
   160    184   			set pkgInfo(hash_type) "sha1"
   161    185   			set pkgInfo(isLatest) [expr {!![lindex $work 5]}]
   162    186   
   163         -			if {[string length $pkgInfo(hash)] != 40} {
          187  +			if {![_isHash $pkgInfo(hash)]} {
   164    188   				continue
   165    189   			}
   166    190   
   167         -			if {![regexp {^[0-9a-f]*$} $pkgInfo(hash)]} {
   168         -				continue
   169         -			}
   170         -
   171         -			set packages($pkgInfo(package)) [array get pkgInfo]
          191  +			lappend curr_packages $pkgInfo(hash)
   172    192   
   173    193   			# Do not do any additional work if we already have this package
   174    194   			set existing_packages [_db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
   175    195   			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
   176    196   				continue
   177    197   			}
   178    198   
   179    199   			if {$pkgInfo(isLatest)} {
   180    200   				_db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo($package) AND os = $pkgInfo($package) AND cpuArch = $pkgInfo(cpuArch);}
   181    201   			}
   182    202   
   183    203   			_db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
          204  +		}
   184    205   
          206  +		# Look for packages that have been deleted
          207  +		set found_packages [_db eval {SELECT sha1 FROM packages WHERE hostname = $hostname;}]
          208  +		foreach package $found_packages {
          209  +			set found_packages_arr($package) 1
          210  +		}
          211  +
          212  +		foreach package $curr_packages {
          213  +			unset -nocomplain found_packages_arr($package)
          214  +		}
          215  +
          216  +		foreach package [array names found_packages_arr] {
          217  +			_db eval {DELETE FROM packages WHERE hostname = $hostname AND sha1 = $package;}
   185    218   		}
   186    219   
   187         -		_db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate) VALUES ($hostname, $now);}
          220  +		_db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::ttl);}
   188    221   
   189    222   		return COMPLETE
   190    223   	}
   191    224   
   192    225   	proc getpkgmanifest {hostname package_sha1} {
   193    226   		set haveManifests [_db eval {SELECT haveManifest FROM packages WHERE sha1 = $package_sha1 LIMIT 1;}]
   194    227   		set haveManifest [lindex $haveManifests 0]
   195    228   
   196    229   		if {$haveManifest} {
   197    230   			return COMPLETE
   198    231   		}
          232  +
          233  +		if {![_isHash $package_sha1]} {
          234  +			return FAIL
          235  +		}
   199    236   
   200    237   		set file [download $hostname $package_sha1]
   201    238   		set fd [open $file]
   202    239   		set pkgdata [read $fd]
   203    240   		close $fd
   204    241   
   205    242   		foreach line [split $pkgdata "\n"] {