AppFS
Changes On Branch complete-multihash
Login
HomeTimelineFilesBranchesTagsTicketsWiki
Hide Diff Patch Side-by-Side Diff

Changes In Branch complete-multihash Excluding Merge-Ins

This is equivalent to a diff from 3864351ad4 to bfaf95df49

2017-02-06
15:57
Added support for a Tcl "exec" target to be compiled in check-in: 3d73fc5750 user: rkeene tags: trunk
2017-01-20
17:19
More work towards actually doing multiple hashing algorithms Leaf check-in: bfaf95df49 user: rkeene tags: complete-multihash
2017-01-19
17:49
Started work on completely supporting multiple hashing algorithms check-in: 2460a1ddab user: rkeene tags: complete-multihash
2016-07-10
19:26
Added support for allowing the user to completely control how downloads are performed as well as configure the default method check-in: 3864351ad4 user: rkeene tags: trunk
19:05
Fixed typo in debug message check-in: 936a791a4a user: rkeene tags: trunk

Modified README.md from [35b476a0b0] to [a25b7811db].

︙
14
15
16
17
18
19
20
21
22


23
24
25
26


27
28
29
30
31

32
33
34


35
36
37

38
39


40
41
42
43

14
15
16
17
18
19
20


21
22
23
24


25
26
27
28
29
30

31
32


33
34
35
36
37
38


39
40
41
42
43
44








-
-
+
+


-
-
+
+




-
+

-
-
+
+



+
-
-
+
+





    AppFS should normally be mounted on "/opt/appfs".

    /opt/appfs/hostname
    	Fetches: http://hostname/appfs/index
    	Contains CSV file: hash,hashMethod,<certificateInDERFormatInHex>,<PKCS#1v1.5-signature-inDERFormatInHex>
	                   \-------------/
                                  ^- Signed data
    	Fetches: http://hostname/appfs/sha1/<hash>
    	Contains CSV file: package,version,os,cpuArch,sha1,isLatest
    	Fetches: http://hostname/appfs/<hashMethod>/<hash>
    	Contains CSV file: package,version,os,cpuArch,packageManifestHash,isLatest

    /opt/appfs/hostname/package/os-cpuArch/version
    /opt/appfs/hostname/sha1/
    	Fetches: http://hostname/appfs/sha1/<sha1>
    /opt/appfs/hostname/<hashMethod>/
    	Fetches: http://hostname/appfs/<hashMethod>/<packageManifestHash>
    	Contains CSV file:
    		type,time,extraData,name
    		type == directory; extraData = (null)
    		type == symlink; extraData = source
    		type == file; extraData = size,perms,sha1
    		type == file; extraData = size,perms,fileHash

    /opt/appfs/hostname/{sha1,package/os-cpuArch/version}/file
    	Fetches: http://hostname/appfs/sha1/<sha1>
    /opt/appfs/hostname/{packageManifestHash,package/os-cpuArch/version}/file
    	Fetches: http://hostname/appfs/<hashMethod>/<fileHash>

Database
--------
    sites(hostname, hashMethod, lastUpdate, ttl)
    packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest)
    files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory)
    packages(hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest)
    files(packageManifestHash, type, time, source, size, perms, fileHash, file_name, file_directory)

Resources
---------
http://appfs.rkeene.org/

Modified appfsd.tcl from [985d2df621] to [fbd0fd32b5].

︙
40
41
42
43
44
45
46
47

48
49
50
51
52
53




54
55
56
57
58
59
60

40
41
42
43
44
45
46

47
48
49
50
51


52
53
54
55
56
57
58
59
60
61
62








-
+




-
-
+
+
+
+









	# User-replaceable function get the home directory of the current user
	proc get_homedir {} {
		return [::appfsd::get_homedir]
	}

	# User-replacable function to update permissions
	proc change_perms {file sha1 perms} {
	proc change_perms {file hash perms {hashMethod "sha1"}} {
		if {[info exists ::appfs::user::add_perms($file)]} {
			append perms $::appfs::user::add_perms($file)
		}

		if {[info exists ::appfs::user::add_perms($sha1)]} {
			append perms $::appfs::user::add_perms($sha1)
		if {[info exists ::appfs::user::add_perms([list $hashMethod $hash])]} {
			append perms $::appfs::user::add_perms([list $hashMethod $hash])
		} elseif {$hashMethod eq "sha1" && [info exists ::appfs::user::add_perms($hash)]} {
			append perms $::appfs::user::add_perms($hash)
		}

		return $perms
	}

	# User-replacable function to fetch a remote file
	proc download_file {url {outputChannel ""}} {
︙
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182

183
184
185
186
187
188
189
190
191

192
193
194
195
196
197
198

166
167
168
169
170
171
172




173
174
175
176
177
178
179

180
181
182
183
184
185
186
187
188

189
190
191
192
193
194
195
196








-
-
-
-







-
+








-
+








		return $file
	}


	proc _isHash {value} {
		set value [string tolower $value]

		if {[string length $value] != 40} {
			return false
		}

		if {![regexp {^[0-9a-f]*$} $value]} {
			return false
		}

		return true
	}

	proc _verifySignatureAndCertificate {hostname certificate signature hash} {
	proc _verifySignatureAndCertificate {hostname certificate signature hash hashmethod} {
		set certificate [binary format "H*" $certificate]
		set signature   [binary format "H*" $signature]

		set certificate [::pki::x509::parse_cert $certificate]

		array set certificate_arr $certificate
		set certificate_cn [::pki::x509::_dn_to_cn $certificate_arr(subject)]

		if {![::pki::verify $signature "$hash,sha1" $certificate]} {
		if {![::pki::verify $signature "$hash,$hashmethod" $certificate]} {
			return false
		}

		if {[string tolower $certificate_cn] != [string tolower $hostname]} {
			return false
		}
︙
308
309
310
311
312
313
314
315
316
317



318
319
320
321
322


323
324
325
326
327
328
329

306
307
308
309
310
311
312



313
314
315
316
317
318


319
320
321
322
323
324
325
326
327








-
-
-
+
+
+



-
-
+
+









			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]

			::appfs::db timeout 30000
		}

		# Create tables
		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate, ttl);}
		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest);}
		db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory);}
		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, hashMethod, lastUpdate, ttl);}
		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest);}
		db eval {CREATE TABLE IF NOT EXISTS files(packageManifestHash, hashMethod, type, time, source, size, perms, fileHash, file_name, file_directory);}

		# Create indexes
		db eval {CREATE INDEX IF NOT EXISTS sites_index ON sites (hostname);}
		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, sha1, package, version, os, cpuArch);}
		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (package_sha1, file_name, file_directory);}
		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, packageManifestHash, package, version, os, cpuArch);}
		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (packageManifestHash, file_name, file_directory);}
	}

	proc download {hostname hash {method sha1}} {
		set url [::appfs::user::construct_url $hostname $hash $method]
		set file [_cachefile $url $hash $method]

		if {![file exists $file]} {
︙
374
375
376
377
378
379
380
381

382
383
384
385

386
387
388
389
390
391
392

372
373
374
375
376
377
378

379
380
381
382

383
384
385
386
387
388
389
390








-
+



-
+








		set indexhashcert   [lindex $indexhash_data 2]
		set indexhashsig    [lindex $indexhash_data 3]

		if {![_isHash $indexhash]} {
			return -code error "Invalid hash: $indexhash"
		}

		if {![_verifySignatureAndCertificate $hostname $indexhashcert $indexhashsig $indexhash]} {
		if {![_verifySignatureAndCertificate $hostname $indexhashcert $indexhashsig $indexhash $indexhashmethod]} {
			return -code error "Invalid signature or certificate from $hostname"
		}

		set file [download $hostname $indexhash]
		set file [download $hostname $indexhash $indexhashmethod]
		catch {
			set fd [open $file]
		}

		if {![info exists fd]} {
			return -code error "Unable to download or open $file"
		}
︙
415
416
417
418
419
420
421
422

423
424
425
426
427
428
429
430
431
432
433
434
435

436
437
438
439
440
441
442
443
444

445
446
447
448
449
450
451
452
453
454
455
456
457
458

459
460
461

462
463
464
465
466
467
468
469



470
471
472
473
474
475

476
477
478
479

480
481
482
483
484
485
486

413
414
415
416
417
418
419

420
421
422
423
424
425
426
427
428
429
430
431
432

433
434
435
436
437
438
439
440
441

442
443
444
445
446
447
448
449
450
451
452
453
454
455

456
457
458

459
460
461
462
463
464
465


466
467
468
469
470
471
472
473

474
475
476
477

478
479
480
481
482
483
484
485








-
+












-
+








-
+













-
+


-
+






-
-
+
+
+





-
+



-
+








			unset -nocomplain pkgInfo
			if {[catch {
				set pkgInfo(package)  [lindex $work 0]
				set pkgInfo(version)  [lindex $work 1]
				set pkgInfo(os)       [_normalizeOS [lindex $work 2]]
				set pkgInfo(cpuArch)  [_normalizeCPU [lindex $work 3]]
				set pkgInfo(hash)     [string tolower [lindex $work 4]]
				set pkgInfo(hash_type) "sha1"
				set pkgInfo(hash_type) $indexhashmethod
				set pkgInfo(isLatest) [expr {!![lindex $work 5]}]
			}]} {
				continue
			}

			if {![_isHash $pkgInfo(hash)]} {
				continue
			}

			lappend curr_packages $pkgInfo(hash)

			# Do not do any additional work if we already have this package
			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND packageManifestHash = $pkgInfo(hash);}]
			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
				continue
			}

			if {$pkgInfo(isLatest)} {
				db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo(package) AND os = $pkgInfo(os) AND cpuArch = $pkgInfo(cpuArch);}
			}

			db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
			db eval {INSERT INTO packages (hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
		}

		# Look for packages that have been deleted
		set found_packages [db eval {SELECT sha1 FROM packages WHERE hostname = $hostname;}]
		foreach package $found_packages {
			set found_packages_arr($package) 1
		}

		foreach package $curr_packages {
			unset -nocomplain found_packages_arr($package)
		}

		foreach package [array names found_packages_arr] {
			db eval {DELETE FROM packages WHERE hostname = $hostname AND sha1 = $package;}
			db eval {DELETE FROM packages WHERE hostname = $hostname AND packageManifestHash = $package;}
		}

		db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::ttl);}
		db eval {INSERT OR REPLACE INTO sites (hostname, hashMethod, lastUpdate, ttl) VALUES ($hostname, $indexhashmethod, $now, $::appfs::ttl);}

		appfsd::get_path_info_cache_flush

		return COMPLETE
	}

	proc getpkgmanifest {hostname package_sha1} {
		set haveManifest [db onecolumn {SELECT haveManifest FROM packages WHERE sha1 = $package_sha1 LIMIT 1;}]
	proc getpkgmanifest {hostname packageManifestHash} {
		set haveManifest [db onecolumn {SELECT haveManifest FROM packages WHERE packageManifestHash = $packageManifestHash LIMIT 1;}]
		set siteHashMethod [db onecolumn {SELECT hashMethod FROM sites WHERE hostname = $hostname LIMIT 1;}

		if {$haveManifest == "1"} {
			return COMPLETE
		}

		if {![_isHash $package_sha1]} {
		if {![_isHash $packageManifestHash]} {
			return FAIL
		}

		set file [download $hostname $package_sha1]
		set file [download $hostname $packageManifestHash $siteHashMethod]

		catch {
			set fd [open $file]
		}

		if {![info exists fd]} {
			return -code error "Unable to download or open $file"
︙