Check-in [bfaf95df49]
Overview
Comment:More work towards actually doing multiple hashing algorithms
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | complete-multihash
Files: files | file ages | folders
SHA1:bfaf95df49a158a4635b394c126717ad880d8f60
User & Date: rkeene on 2017-01-20 17:19:04
Other Links: manifest | tags
Context
2017-01-20
17:19
More work towards actually doing multiple hashing algorithms Leaf check-in: bfaf95df49 user: rkeene tags: complete-multihash
2017-01-19
17:49
Started work on completely supporting multiple hashing algorithms check-in: 2460a1ddab user: rkeene tags: complete-multihash
Changes

Modified README.md from [35b476a0b0] to [a25b7811db].

    14     14       AppFS should normally be mounted on "/opt/appfs".
    15     15   
    16     16       /opt/appfs/hostname
    17     17       	Fetches: http://hostname/appfs/index
    18     18       	Contains CSV file: hash,hashMethod,<certificateInDERFormatInHex>,<PKCS#1v1.5-signature-inDERFormatInHex>
    19     19   	                   \-------------/
    20     20                                     ^- Signed data
    21         -    	Fetches: http://hostname/appfs/sha1/<hash>
    22         -    	Contains CSV file: package,version,os,cpuArch,sha1,isLatest
           21  +    	Fetches: http://hostname/appfs/<hashMethod>/<hash>
           22  +    	Contains CSV file: package,version,os,cpuArch,packageManifestHash,isLatest
    23     23   
    24     24       /opt/appfs/hostname/package/os-cpuArch/version
    25         -    /opt/appfs/hostname/sha1/
    26         -    	Fetches: http://hostname/appfs/sha1/<sha1>
           25  +    /opt/appfs/hostname/<hashMethod>/
           26  +    	Fetches: http://hostname/appfs/<hashMethod>/<packageManifestHash>
    27     27       	Contains CSV file:
    28     28       		type,time,extraData,name
    29     29       		type == directory; extraData = (null)
    30     30       		type == symlink; extraData = source
    31         -    		type == file; extraData = size,perms,sha1
           31  +    		type == file; extraData = size,perms,fileHash
    32     32   
    33         -    /opt/appfs/hostname/{sha1,package/os-cpuArch/version}/file
    34         -    	Fetches: http://hostname/appfs/sha1/<sha1>
           33  +    /opt/appfs/hostname/{packageManifestHash,package/os-cpuArch/version}/file
           34  +    	Fetches: http://hostname/appfs/<hashMethod>/<fileHash>
    35     35   
    36     36   Database
    37     37   --------
    38         -    packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest)
    39         -    files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory)
           38  +    sites(hostname, hashMethod, lastUpdate, ttl)
           39  +    packages(hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest)
           40  +    files(packageManifestHash, type, time, source, size, perms, fileHash, file_name, file_directory)
    40     41   
    41     42   Resources
    42     43   ---------
    43     44   http://appfs.rkeene.org/

Modified appfsd.tcl from [bd58d61afc] to [fbd0fd32b5].

   166    166   		return $file
   167    167   	}
   168    168   
   169    169   
   170    170   	proc _isHash {value} {
   171    171   		set value [string tolower $value]
   172    172   
   173         -		if {[string length $value] != 40} {
   174         -			return false
   175         -		}
   176         -
   177    173   		if {![regexp {^[0-9a-f]*$} $value]} {
   178    174   			return false
   179    175   		}
   180    176   
   181    177   		return true
   182    178   	}
   183    179   
   184         -	proc _verifySignatureAndCertificate {hostname certificate signature hash} {
          180  +	proc _verifySignatureAndCertificate {hostname certificate signature hash hashmethod} {
   185    181   		set certificate [binary format "H*" $certificate]
   186    182   		set signature   [binary format "H*" $signature]
   187    183   
   188    184   		set certificate [::pki::x509::parse_cert $certificate]
   189    185   
   190    186   		array set certificate_arr $certificate
   191    187   		set certificate_cn [::pki::x509::_dn_to_cn $certificate_arr(subject)]
   192    188   
   193         -		if {![::pki::verify $signature "$hash,sha1" $certificate]} {
          189  +		if {![::pki::verify $signature "$hash,$hashmethod" $certificate]} {
   194    190   			return false
   195    191   		}
   196    192   
   197    193   		if {[string tolower $certificate_cn] != [string tolower $hostname]} {
   198    194   			return false
   199    195   		}
   200    196   
................................................................................
   310    306   
   311    307   			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]
   312    308   
   313    309   			::appfs::db timeout 30000
   314    310   		}
   315    311   
   316    312   		# Create tables
   317         -		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate, ttl);}
   318         -		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest);}
   319         -		db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory);}
          313  +		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, hashMethod, lastUpdate, ttl);}
          314  +		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest);}
          315  +		db eval {CREATE TABLE IF NOT EXISTS files(packageManifestHash, hashMethod, type, time, source, size, perms, fileHash, file_name, file_directory);}
   320    316   
   321    317   		# Create indexes
   322    318   		db eval {CREATE INDEX IF NOT EXISTS sites_index ON sites (hostname);}
   323         -		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, sha1, package, version, os, cpuArch);}
   324         -		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (package_sha1, file_name, file_directory);}
          319  +		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, packageManifestHash, package, version, os, cpuArch);}
          320  +		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (packageManifestHash, file_name, file_directory);}
   325    321   	}
   326    322   
   327    323   	proc download {hostname hash {method sha1}} {
   328    324   		set url [::appfs::user::construct_url $hostname $hash $method]
   329    325   		set file [_cachefile $url $hash $method]
   330    326   
   331    327   		if {![file exists $file]} {
................................................................................
   376    372   		set indexhashcert   [lindex $indexhash_data 2]
   377    373   		set indexhashsig    [lindex $indexhash_data 3]
   378    374   
   379    375   		if {![_isHash $indexhash]} {
   380    376   			return -code error "Invalid hash: $indexhash"
   381    377   		}
   382    378   
   383         -		if {![_verifySignatureAndCertificate $hostname $indexhashcert $indexhashsig $indexhash]} {
          379  +		if {![_verifySignatureAndCertificate $hostname $indexhashcert $indexhashsig $indexhash $indexhashmethod]} {
   384    380   			return -code error "Invalid signature or certificate from $hostname"
   385    381   		}
   386    382   
   387         -		set file [download $hostname $indexhash]
          383  +		set file [download $hostname $indexhash $indexhashmethod]
   388    384   		catch {
   389    385   			set fd [open $file]
   390    386   		}
   391    387   
   392    388   		if {![info exists fd]} {
   393    389   			return -code error "Unable to download or open $file"
   394    390   		}
................................................................................
   417    413   			unset -nocomplain pkgInfo
   418    414   			if {[catch {
   419    415   				set pkgInfo(package)  [lindex $work 0]
   420    416   				set pkgInfo(version)  [lindex $work 1]
   421    417   				set pkgInfo(os)       [_normalizeOS [lindex $work 2]]
   422    418   				set pkgInfo(cpuArch)  [_normalizeCPU [lindex $work 3]]
   423    419   				set pkgInfo(hash)     [string tolower [lindex $work 4]]
   424         -				set pkgInfo(hash_type) "sha1"
          420  +				set pkgInfo(hash_type) $indexhashmethod
   425    421   				set pkgInfo(isLatest) [expr {!![lindex $work 5]}]
   426    422   			}]} {
   427    423   				continue
   428    424   			}
   429    425   
   430    426   			if {![_isHash $pkgInfo(hash)]} {
   431    427   				continue
   432    428   			}
   433    429   
   434    430   			lappend curr_packages $pkgInfo(hash)
   435    431   
   436    432   			# Do not do any additional work if we already have this package
   437         -			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
          433  +			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND packageManifestHash = $pkgInfo(hash);}]
   438    434   			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
   439    435   				continue
   440    436   			}
   441    437   
   442    438   			if {$pkgInfo(isLatest)} {
   443    439   				db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo(package) AND os = $pkgInfo(os) AND cpuArch = $pkgInfo(cpuArch);}
   444    440   			}
   445    441   
   446         -			db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
          442  +			db eval {INSERT INTO packages (hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
   447    443   		}
   448    444   
   449    445   		# Look for packages that have been deleted
   450    446   		set found_packages [db eval {SELECT sha1 FROM packages WHERE hostname = $hostname;}]
   451    447   		foreach package $found_packages {
   452    448   			set found_packages_arr($package) 1
   453    449   		}
   454    450   
   455    451   		foreach package $curr_packages {
   456    452   			unset -nocomplain found_packages_arr($package)
   457    453   		}
   458    454   
   459    455   		foreach package [array names found_packages_arr] {
   460         -			db eval {DELETE FROM packages WHERE hostname = $hostname AND sha1 = $package;}
          456  +			db eval {DELETE FROM packages WHERE hostname = $hostname AND packageManifestHash = $package;}
   461    457   		}
   462    458   
   463         -		db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::ttl);}
          459  +		db eval {INSERT OR REPLACE INTO sites (hostname, hashMethod, lastUpdate, ttl) VALUES ($hostname, $indexhashmethod, $now, $::appfs::ttl);}
   464    460   
   465    461   		appfsd::get_path_info_cache_flush
   466    462   
   467    463   		return COMPLETE
   468    464   	}
   469    465   
   470         -	proc getpkgmanifest {hostname package_sha1} {
   471         -		set haveManifest [db onecolumn {SELECT haveManifest FROM packages WHERE sha1 = $package_sha1 LIMIT 1;}]
          466  +	proc getpkgmanifest {hostname packageManifestHash} {
          467  +		set haveManifest [db onecolumn {SELECT haveManifest FROM packages WHERE packageManifestHash = $packageManifestHash LIMIT 1;}]
          468  +		set siteHashMethod [db onecolumn {SELECT hashMethod FROM sites WHERE hostname = $hostname LIMIT 1;}
   472    469   
   473    470   		if {$haveManifest == "1"} {
   474    471   			return COMPLETE
   475    472   		}
   476    473   
   477         -		if {![_isHash $package_sha1]} {
          474  +		if {![_isHash $packageManifestHash]} {
   478    475   			return FAIL
   479    476   		}
   480    477   
   481         -		set file [download $hostname $package_sha1]
          478  +		set file [download $hostname $packageManifestHash $siteHashMethod]
   482    479   
   483    480   		catch {
   484    481   			set fd [open $file]
   485    482   		}
   486    483   
   487    484   		if {![info exists fd]} {
   488    485   			return -code error "Unable to download or open $file"