Check-in [48a551bede]
Overview
Comment:Updated to populate SQLite database with all package data
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:48a551bede7cc48fce7871f033ad1dd89e5ead6b
User & Date: rkeene on 2014-09-08 02:33:28
Other Links: manifest | tags
Context
2014-09-08
02:38
Separated manifest fetching from index fetching check-in: b5d1616f0f user: rkeene tags: trunk
02:33
Updated to populate SQLite database with all package data check-in: 48a551bede user: rkeene tags: trunk
01:24
More work on basics check-in: a189f64907 user: rkeene tags: trunk
Changes

Modified README.md from [5b04e66b64] to [15ef741d8b].

    23     23       		type == file; extraData = size,sha1
    24     24   
    25     25       /opt/appfs/hostname/{sha1,package/os-cpuArch/version}/file
    26     26       	Fetches: http://hostname/appfs/sha1/<sha1>
    27     27   
    28     28   Database
    29     29   --------
    30         -    hostname_to_packages(hostname, sha1, package, version, os, cpuArch, isLatest)
    31         -    package_to_files(package_sha1, type, time, source, size, file_sha1, name)
           30  +    packages(hostname, sha1, package, version, os, cpuArch, isLatest)
           31  +    files(package_sha1, type, time, source, size, file_sha1, file_name, file_directory)

Modified appfs.c from [1eae70cf2b] to [4b3327c544].

     1      1   #define FUSE_USE_VERSION 26
     2      2   
     3      3   #include <string.h>
            4  +#include <stdarg.h>
     4      5   #include <errno.h>
     5      6   #include <fcntl.h>
     6      7   #include <stdio.h>
     7      8   #include <fuse.h>
     8      9   #include <tcl.h>
     9     10   
    10     11   #define APPFS_DEBUG(x...) { fprintf(stderr, "%i:%s: ", __LINE__, __func__); fprintf(stderr, x); fprintf(stderr, "\n"); }
................................................................................
   110    111   			return("arm");
   111    112   		case APPFS_CPU_UNKNOWN:
   112    113   			return("unknown");
   113    114   	}
   114    115   
   115    116   	return("unknown");
   116    117   }
          118  +
          119  +static int appfs_Tcl_Eval(Tcl_Interp *interp, int objc, const char *cmd, ...) {
          120  +	Tcl_Obj **objv;
          121  +	const char *arg;
          122  +	va_list argp;
          123  +	int retval;
          124  +	int i;
          125  +
          126  +	objv = ckalloc(sizeof(*objv) * objc);
          127  +	objv[0] = Tcl_NewStringObj(cmd, -1);
          128  +
          129  +	va_start(argp, cmd);
          130  +	for (i = 1; i < objc; i++) {
          131  +		arg = va_arg(argp, const char *);
          132  +		objv[i] = Tcl_NewStringObj(arg, -1);
          133  +	}
          134  +	va_end(argp);
          135  +
          136  +	retval = Tcl_EvalObjv(interp, objc, objv, 0);
          137  +
          138  +	ckfree(objv);
          139  +
          140  +	return(retval);
          141  +}
   117    142   
   118    143   static struct appfs_package *appfs_getindex(const char *hostname, int *package_count_p) {
   119         -	Tcl_Obj *objv[2];
   120         -	Tcl_Obj *packages_tcl;
   121    144   	int tcl_ret;
   122    145   
   123    146   	if (package_count_p == NULL) {
   124    147   		return(NULL);
   125    148   	}
   126    149   
   127         -	objv[0] = Tcl_NewStringObj("::appfs::getindex", -1);
   128         -	objv[1] = Tcl_NewStringObj(hostname, -1);
   129         -
   130         -	tcl_ret = Tcl_EvalObjv(interp, 2, objv, 0);
          150  +	tcl_ret = appfs_Tcl_Eval(interp, 2, "::appfs::getindex", hostname);
   131    151   	if (tcl_ret != TCL_OK) {
   132    152   		APPFS_DEBUG("Call to ::appfs::getindex failed: %s", Tcl_GetStringResult(interp));
   133    153   
   134    154   		return(NULL);
   135    155   	}
   136    156   
   137         -	packages_tcl = Tcl_GetObjResult(interp);
   138         -
   139    157   	return(NULL);
   140    158   }
   141    159   
   142    160   static int appfs_getfile(const char *hostname, const char *sha1) {
   143    161   }
   144    162   
   145    163   static int appfs_getmanifest(const char *hostname, const char *sha1) {
................................................................................
   215    233   		return(1);
   216    234   	}
   217    235   
   218    236   	tcl_ret = Tcl_Eval(interp, ""
   219    237   #include "appfs.tcl.h"
   220    238   	"");
   221    239   	if (tcl_ret != TCL_OK) {
   222         -		fprintf(stderr, "Unable to initialize Tcl AppFS Script.  Aborting.\n");
          240  +		fprintf(stderr, "Unable to initialize Tcl AppFS script.  Aborting.\n");
          241  +
          242  +		return(1);
          243  +	}
          244  +
          245  +	tcl_ret = appfs_Tcl_Eval(interp, 1, "::appfs::init");
          246  +	if (tcl_ret != TCL_OK) {
          247  +		fprintf(stderr, "Unable to initialize Tcl AppFS script (::appfs::init).  Aborting.\n");
          248  +		fprintf(stderr, "Tcl Error is: %s\n", Tcl_GetStringResult(interp));
   223    249   
   224    250   		return(1);
   225    251   	}
   226    252   
   227    253   #ifdef APPFS_TEST_DRIVER
   228    254   	return(appfs_test_driver());
   229    255   #else
   230    256   	return(fuse_main(argc, argv, &appfs_oper, NULL));
   231    257   #endif
   232    258   }
   233    259    

Modified appfs.tcl from [07f0ced01b] to [58a43bb4f1].

     1      1   #! /usr/bin/env tclsh
     2      2   
     3      3   package require http 2.7
     4      4   package require sqlite3
     5      5   
     6      6   namespace eval ::appfs {
     7         -	variable sites [list]
     8      7   	variable cachedir "/tmp/appfs-cache"
     9      8   
    10      9   	proc _hash_sep {hash {seps 4}} {
    11     10   		for {set idx 0} {$idx < $seps} {incr idx} {
    12     11   			append retval "[string range $hash [expr {$idx * 2}] [expr {($idx * 2) + 1}]]/"
    13     12   		}
    14     13   		append retval "[string range $hash [expr {$idx * 2}] end]"
................................................................................
    40     39   			} else {
    41     40   				file delete -force -- $tmpfile
    42     41   			}
    43     42   		}
    44     43   
    45     44   		return $file
    46     45   	}
           46  +
           47  +	proc _db {args} {
           48  +		return [uplevel 1 [list ::appfs::db {*}$args]]
           49  +	}
           50  +
           51  +	proc init {} {
           52  +		if {[info exists ::appfs::init_called]} {
           53  +			return
           54  +		}
           55  +
           56  +		set ::appfs::init_called 1
           57  +
           58  +		if {![info exists ::appfs::db]} {
           59  +			file mkdir $::appfs::cachedir
           60  +
           61  +			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]
           62  +		}
           63  +
           64  +		_db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest);}
           65  +		_db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, file_sha1, file_name, file_directory);}
           66  +	}
    47     67   
    48     68   	proc getindex {hostname} {
    49     69   		if {[string match "*\[/~\]*" $hostname]} {
    50     70   			return -code error "Invalid hostname"
    51     71   		}
    52     72   
    53     73   		set url "http://$hostname/appfs/index"
................................................................................
    96    116   			}
    97    117   
    98    118   			if {![regexp {^[0-9a-f]*$} $pkgInfo(hash)]} {
    99    119   				continue
   100    120   			}
   101    121   
   102    122   			set packages($pkgInfo(package)) [array get pkgInfo]
          123  +
          124  +			# Do not do any additional work if we already have this package
          125  +			set existing_packages [_db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
          126  +			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
          127  +				continue
          128  +			}
          129  +
          130  +			if {$pkgInfo(isLatest)} {
          131  +				_db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo($package) AND os = $pkgInfo($package) AND cpuArch = $pkgInfo(cpuArch);}
          132  +			}
          133  +
          134  +			_db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest) );}
          135  +
          136  +			set file [download $hostname $pkgInfo(hash)]
          137  +			set fd [open $file]
          138  +			set pkgdata [read $fd]
          139  +			close $fd
          140  +
          141  +			foreach line [split $pkgdata "\n"] {
          142  +				set line [string trim $line]
          143  +
          144  +				if {[string match "*/*" $line]} {
          145  +					continue
          146  +				}
          147  +
          148  +				if {$line == ""} {
          149  +					continue
          150  +				}
          151  +
          152  +				set work [split $line ","]
          153  +
          154  +				unset -nocomplain fileInfo
          155  +				set fileInfo(type) [lindex $work 0]
          156  +				set fileInfo(time) [lindex $work 1]
          157  +				set fileInfo(name) [lindex $work end]
          158  +
          159  +				set fileInfo(name) [split [string trim $fileInfo(name) "/"] "/"]
          160  +				set fileInfo(directory) [join [lrange $fileInfo(name) 0 end-1] "/"]
          161  +				set fileInfo(name) [lindex $fileInfo(name) end]
          162  +
          163  +				set work [lrange $work 2 end-1]
          164  +				switch -- $fileInfo(type) {
          165  +					"file" {
          166  +						set fileInfo(size) [lindex $work 0]
          167  +						set fileInfo(sha1) [lindex $work 1]
          168  +					}
          169  +					"symlink" {
          170  +						set fileInfo(source) [lindex $work 0]
          171  +					}
          172  +				}
          173  +
          174  +				_db eval {INSERT INTO files (package_sha1, type, time, source, size, file_sha1, file_name, file_directory) VALUES ($pkgInfo(hash), $fileInfo(type), $fileInfo(time), $fileInfo(source), $fileInfo(size), $fileInfo(sha1), $fileInfo(name), $fileInfo(directory) );}
          175  +			}
   103    176   		}
   104    177   
   105         -		return [array get packages]
          178  +		return COMPLETE
   106    179   	}
   107    180   
   108    181   	proc download {hostname hash {method sha1}} {
   109    182   		set url "http://$hostname/appfs/$method/$hash"
   110    183   		set file [_cachefile $url $hash]
   111    184   
   112    185   		if {![file exists $file]} {
   113    186   			return -code error "Unable to fetch"
   114    187   		}
   115    188   
   116    189   		return $file
   117    190   	}
   118    191   }