Check-in [b5d1616f0f]
Overview
Comment:Separated manifest fetching from index fetching
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:b5d1616f0f6849fe2ffa99c58ae81a49bba57b20
User & Date: rkeene on 2014-09-08 02:38:06
Other Links: manifest | tags
Context
2014-09-08
02:50
Updated with support for not redownloading manifest if it's already present check-in: d64cb6e110 user: rkeene tags: trunk
02:38
Separated manifest fetching from index fetching check-in: b5d1616f0f user: rkeene tags: trunk
02:33
Updated to populate SQLite database with all package data check-in: 48a551bede user: rkeene tags: trunk
Changes

Modified appfs.tcl from [58a43bb4f1] to [b782d4f8ad].

    60     60   
    61     61   			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]
    62     62   		}
    63     63   
    64     64   		_db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest);}
    65     65   		_db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, file_sha1, file_name, file_directory);}
    66     66   	}
           67  +
           68  +	proc download {hostname hash {method sha1}} {
           69  +		set url "http://$hostname/appfs/$method/$hash"
           70  +		set file [_cachefile $url $hash]
           71  +
           72  +		if {![file exists $file]} {
           73  +			return -code error "Unable to fetch"
           74  +		}
           75  +
           76  +		return $file
           77  +	}
    67     78   
    68     79   	proc getindex {hostname} {
    69     80   		if {[string match "*\[/~\]*" $hostname]} {
    70     81   			return -code error "Invalid hostname"
    71     82   		}
    72     83   
    73     84   		set url "http://$hostname/appfs/index"
................................................................................
   129    140   
   130    141   			if {$pkgInfo(isLatest)} {
   131    142   				_db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo($package) AND os = $pkgInfo($package) AND cpuArch = $pkgInfo(cpuArch);}
   132    143   			}
   133    144   
   134    145   			_db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest) );}
   135    146   
   136         -			set file [download $hostname $pkgInfo(hash)]
   137         -			set fd [open $file]
   138         -			set pkgdata [read $fd]
   139         -			close $fd
   140         -
   141         -			foreach line [split $pkgdata "\n"] {
   142         -				set line [string trim $line]
   143         -
   144         -				if {[string match "*/*" $line]} {
   145         -					continue
   146         -				}
   147         -
   148         -				if {$line == ""} {
   149         -					continue
   150         -				}
   151         -
   152         -				set work [split $line ","]
   153         -
   154         -				unset -nocomplain fileInfo
   155         -				set fileInfo(type) [lindex $work 0]
   156         -				set fileInfo(time) [lindex $work 1]
   157         -				set fileInfo(name) [lindex $work end]
   158         -
   159         -				set fileInfo(name) [split [string trim $fileInfo(name) "/"] "/"]
   160         -				set fileInfo(directory) [join [lrange $fileInfo(name) 0 end-1] "/"]
   161         -				set fileInfo(name) [lindex $fileInfo(name) end]
   162         -
   163         -				set work [lrange $work 2 end-1]
   164         -				switch -- $fileInfo(type) {
   165         -					"file" {
   166         -						set fileInfo(size) [lindex $work 0]
   167         -						set fileInfo(sha1) [lindex $work 1]
   168         -					}
   169         -					"symlink" {
   170         -						set fileInfo(source) [lindex $work 0]
   171         -					}
   172         -				}
   173         -
   174         -				_db eval {INSERT INTO files (package_sha1, type, time, source, size, file_sha1, file_name, file_directory) VALUES ($pkgInfo(hash), $fileInfo(type), $fileInfo(time), $fileInfo(source), $fileInfo(size), $fileInfo(sha1), $fileInfo(name), $fileInfo(directory) );}
   175         -			}
   176    147   		}
   177    148   
   178    149   		return COMPLETE
   179    150   	}
   180    151   
   181         -	proc download {hostname hash {method sha1}} {
   182         -		set url "http://$hostname/appfs/$method/$hash"
   183         -		set file [_cachefile $url $hash]
          152  +	proc getpkgmanifest {hostname package_sha1} {
          153  +		set file [download $hostname $package_sha1]
          154  +		set fd [open $file]
          155  +		set pkgdata [read $fd]
          156  +		close $fd
          157  +
          158  +		foreach line [split $pkgdata "\n"] {
          159  +			set line [string trim $line]
          160  +
          161  +			if {[string match "*/*" $line]} {
          162  +				continue
          163  +			}
          164  +
          165  +			if {$line == ""} {
          166  +				continue
          167  +			}
          168  +
          169  +			set work [split $line ","]
          170  +
          171  +			unset -nocomplain fileInfo
          172  +			set fileInfo(type) [lindex $work 0]
          173  +			set fileInfo(time) [lindex $work 1]
          174  +			set fileInfo(name) [lindex $work end]
          175  +
          176  +			set fileInfo(name) [split [string trim $fileInfo(name) "/"] "/"]
          177  +			set fileInfo(directory) [join [lrange $fileInfo(name) 0 end-1] "/"]
          178  +			set fileInfo(name) [lindex $fileInfo(name) end]
   184    179   
   185         -		if {![file exists $file]} {
   186         -			return -code error "Unable to fetch"
   187         -		}
          180  +			set work [lrange $work 2 end-1]
          181  +			switch -- $fileInfo(type) {
          182  +				"file" {
          183  +					set fileInfo(size) [lindex $work 0]
          184  +					set fileInfo(sha1) [lindex $work 1]
          185  +				}
          186  +				"symlink" {
          187  +					set fileInfo(source) [lindex $work 0]
          188  +				}
          189  +			}
   188    190   
   189         -		return $file
          191  +			_db eval {INSERT INTO files (package_sha1, type, time, source, size, file_sha1, file_name, file_directory) VALUES ($package_sha1, $fileInfo(type), $fileInfo(time), $fileInfo(source), $fileInfo(size), $fileInfo(sha1), $fileInfo(name), $fileInfo(directory) );}
          192  +		}
   190    193   	}
          194  +
   191    195   }