Check-in [c374111c37]
Overview
Comment:Added support for an "appfs-cache" script calling sqlite3 directly in appfsd
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | tcl-ops
Files: files | file ages | folders
SHA1:c374111c3789989554f0143845cdd829a583ec5c
User & Date: rkeene on 2014-11-07 04:52:29
Other Links: manifest | tags
Context
2014-11-07
05:06
Updated to include a Tcl interface via AppFSd check-in: 4b04c967f7 user: rkeene tags: tcl-ops
04:52
Added support for an "appfs-cache" script calling sqlite3 directly in appfsd check-in: c374111c37 user: rkeene tags: tcl-ops
2014-11-06
18:29
Minor update check-in: 5bd7399e05 user: rkeene tags: tcl-ops
Changes

Added appfs-cache version [8c72230216].

            1  +#! /usr/bin/env bash
            2  +
            3  +case "$1" in
            4  +	invalidate)
            5  +		statement='UPDATE sites SET ttl = "0";'
            6  +		;;
            7  +	clear)
            8  +		rm -rf 
            9  +		statement='DELETE FROM sites; DELETE FROM packages; DELETE FROM files;'
           10  +		;;
           11  +	*)
           12  +		echo "Usage: appfs-cache {invalidate|clear}" >&2
           13  +
           14  +		exit 1
           15  +		;;
           16  +esac
           17  +
           18  +exec appfsd -sqlite3 "${statement}"

Modified appfsd.c from [169ff6cab0] to [206ee97d58].

    69     69   		struct {
    70     70   			off_t size;
    71     71   			char source[256];
    72     72   		} symlink;
    73     73   	} typeinfo;
    74     74   };
    75     75   
    76         -static Tcl_Interp *appfs_create_TclInterp(const char *cachedir) {
           76  +static Tcl_Interp *appfs_create_TclInterp(void) {
    77     77   	Tcl_Interp *interp;
           78  +	const char *cachedir = globalThread.cachedir;
    78     79   	int tcl_ret;
    79     80   
    80     81   	APPFS_DEBUG("Creating new Tcl interpreter for TID = 0x%llx", (unsigned long long) pthread_self());
    81     82   
    82     83   	interp = Tcl_CreateInterp();
    83     84   	if (interp == NULL) {
    84     85   		fprintf(stderr, "Unable to create Tcl Interpreter.  Aborting.\n");
................................................................................
   197    198   	Tcl_Interp *interp;
   198    199   	int tcl_ret;
   199    200   
   200    201   	APPFS_DEBUG("Enter: hostname = %s", hostname);
   201    202   
   202    203   	interp = pthread_getspecific(interpKey);
   203    204   	if (interp == NULL) {
   204         -		interp = appfs_create_TclInterp(globalThread.cachedir);
          205  +		interp = appfs_create_TclInterp();
   205    206   
   206    207   		if (interp == NULL) {
   207    208   			return;
   208    209   		}
   209    210   
   210    211   		pthread_setspecific(interpKey, interp);
   211    212   	}
................................................................................
   223    224   static const char *appfs_getfile(const char *hostname, const char *sha1) {
   224    225   	Tcl_Interp *interp;
   225    226   	char *retval;
   226    227   	int tcl_ret;
   227    228   
   228    229   	interp = pthread_getspecific(interpKey);
   229    230   	if (interp == NULL) {
   230         -		interp = appfs_create_TclInterp(globalThread.cachedir);
          231  +		interp = appfs_create_TclInterp();
   231    232   
   232    233   		if (interp == NULL) {
   233    234   			return(NULL);
   234    235   		}
   235    236   
   236    237   		pthread_setspecific(interpKey, interp);
   237    238   	}
................................................................................
   250    251   
   251    252   static void appfs_update_manifest(const char *hostname, const char *sha1) {
   252    253   	Tcl_Interp *interp;
   253    254   	int tcl_ret;
   254    255   
   255    256   	interp = pthread_getspecific(interpKey);
   256    257   	if (interp == NULL) {
   257         -		interp = appfs_create_TclInterp(globalThread.cachedir);
          258  +		interp = appfs_create_TclInterp();
   258    259   
   259    260   		if (interp == NULL) {
   260    261   			return;
   261    262   		}
   262    263   
   263    264   		pthread_setspecific(interpKey, interp);
   264    265   	}
................................................................................
   379    380   
   380    381   	return(retval);
   381    382   }
   382    383   
   383    384   /* Get information about a path, and optionally list children */
   384    385   static int appfs_get_path_info(const char *_path, struct appfs_pathinfo *pathinfo, struct appfs_children **children) {
   385    386   }
          387  +
          388  +static int appfs_fuse_readlink(const char *path, char *buf, size_t size) {
          389  +	struct appfs_pathinfo pathinfo;
          390  +	int res = 0;
          391  +
          392  +	APPFS_DEBUG("Enter (path = %s, ...)", path);
          393  +
          394  +	pathinfo.type = APPFS_PATHTYPE_INVALID;
          395  +
          396  +	res = appfs_get_path_info(path, &pathinfo, NULL);
          397  +	if (res != 0) {
          398  +		return(res);
          399  +	}
          400  +
          401  +	if (pathinfo.type != APPFS_PATHTYPE_SYMLINK) {
          402  +		return(-EINVAL);
          403  +	}
          404  +
          405  +	if ((strlen(pathinfo.typeinfo.symlink.source) + 1) > size) {
          406  +		return(-ENAMETOOLONG);
          407  +	}
          408  +
          409  +	memcpy(buf, pathinfo.typeinfo.symlink.source, strlen(pathinfo.typeinfo.symlink.source) + 1);
          410  +
          411  +	return(0);
          412  +}
   386    413   
   387    414   static int appfs_fuse_getattr(const char *path, struct stat *stbuf) {
   388    415   	struct appfs_pathinfo pathinfo;
   389    416   	int res = 0;
   390    417   
   391    418   	APPFS_DEBUG("Enter (path = %s, ...)", path);
   392    419   
................................................................................
   455    482   	filler(buf, ".", NULL, 0);
   456    483   	filler(buf, "..", NULL, 0);
   457    484   
   458    485   	for (child = children; child; child = child->_next) {
   459    486   		filler(buf, child->name, NULL, 0);
   460    487   	}
   461    488   
   462         -	appfs_free_list_children(children);
          489  +//	appfs_free_list_children(children);
   463    490   
   464    491   	return(0);
   465    492   }
   466    493   
   467    494   static int appfs_fuse_open(const char *path, struct fuse_file_info *fi) {
   468    495   	struct appfs_pathinfo pathinfo;
   469    496   	const char *real_path;
   470    497   	int fh;
   471    498   	int gpi_ret;
   472    499   
   473    500   	APPFS_DEBUG("Enter (path = %s, ...)", path);
          501  +
          502  +#if 0
   474    503   
   475    504   	if ((fi->flags & 3) != O_RDONLY) {
   476    505                   return(-EACCES);
   477    506   	}
   478    507   
   479    508   	gpi_ret = appfs_get_path_info(path, &pathinfo, NULL);
   480    509   	if (gpi_ret != 0) {
................................................................................
   493    522   	fh = open(real_path, O_RDONLY);
   494    523   	free((void *) real_path);
   495    524   	if (fh < 0) {
   496    525   		return(-EIO);
   497    526   	}
   498    527   
   499    528   	fi->fh = fh;
          529  +#endif
   500    530   
   501    531   	return(0);
   502    532   }
   503    533   
   504    534   static int appfs_fuse_close(const char *path, struct fuse_file_info *fi) {
   505    535   	int close_ret;
   506    536   
................................................................................
   524    554   	}
   525    555   
   526    556   	read_ret = read(fi->fh, buf, size);
   527    557   
   528    558   	return(read_ret);
   529    559   }
   530    560   
   531         -static struct fuse_operations appfs_oper = {
   532         -	.getattr   = appfs_fuse_getattr,
   533         -	.readdir   = appfs_fuse_readdir,
   534         -	.readlink  = appfs_fuse_readlink,
   535         -	.open      = appfs_fuse_open,
   536         -	.release   = appfs_fuse_close,
   537         -	.read      = appfs_fuse_read
   538         -};
          561  +static int appfs_sqlite3(const char *sql) {
          562  +	Tcl_Interp *interp;
          563  +	const char *sql_ret;
          564  +	int tcl_ret;
          565  +
          566  +	interp = appfs_create_TclInterp();
          567  +	if (interp == NULL) {
          568  +		fprintf(stderr, "Unable to create a Tcl interpreter.  Aborting.\n");
          569  +
          570  +		return(1);
          571  +	}
          572  +
          573  +	tcl_ret = appfs_Tcl_Eval(interp, 5, "::appfs::db", "eval", sql, "row", "unset -nocomplain row(*); parray row; puts \"----\"");
          574  +	sql_ret = Tcl_GetStringResult(interp);
          575  +
          576  +	if (tcl_ret != TCL_OK) {
          577  +		fprintf(stderr, "[error] %s\n", sql_ret);
          578  +
          579  +		return(1);
          580  +	}
          581  +
          582  +	printf("%s\n", sql_ret);
          583  +
          584  +	return(0);
          585  +}
   539    586   
   540         -int Appfsd_Init(Tcl_Interp *interp) {
          587  +static int Appfsd_Init(Tcl_Interp *interp) {
   541    588   #ifdef USE_TCL_STUBS
   542    589   	if (Tcl_InitStubs(interp, TCL_VERSION, 0) == 0L) {
   543    590   		return(TCL_ERROR);
   544    591   	}
   545    592   #endif
   546    593   
   547    594   	Tcl_CreateObjCommand(interp, "appfsd::get_homedir", tcl_appfs_get_homedir, NULL, NULL);
   548    595   
   549    596   	Tcl_PkgProvide(interp, "appfsd", "1.0");
   550    597   
   551    598   	return(TCL_OK);
   552    599   }
          600  +
          601  +static struct fuse_operations appfs_oper = {
          602  +	.getattr   = appfs_fuse_getattr,
          603  +	.readdir   = appfs_fuse_readdir,
          604  +	.readlink  = appfs_fuse_readlink,
          605  +	.open      = appfs_fuse_open,
          606  +	.release   = appfs_fuse_close,
          607  +	.read      = appfs_fuse_read
          608  +};
   553    609   
   554    610   int main(int argc, char **argv) {
   555    611   	const char *cachedir = APPFS_CACHEDIR;
   556         -	char dbfilename[1024];
   557         -	int pthread_ret, snprintf_ret;
          612  +	int pthread_ret;
   558    613   
   559    614   	globalThread.cachedir = cachedir;
   560    615   	globalThread.boottime = time(NULL);
   561    616   	globalThread.options.writable = 1;
   562    617   
   563    618   	Tcl_StaticPackage(NULL, "sha1", Sha1_Init, NULL);
   564    619   	Tcl_StaticPackage(NULL, "appfsd", Appfsd_Init, NULL);
................................................................................
   566    621   	pthread_ret = pthread_key_create(&interpKey, NULL);
   567    622   	if (pthread_ret != 0) {
   568    623   		fprintf(stderr, "Unable to create TSD key for Tcl.  Aborting.\n");
   569    624   
   570    625   		return(1);
   571    626   	}
   572    627   
   573         -	snprintf_ret = snprintf(dbfilename, sizeof(dbfilename), "%s/%s", cachedir, "cache.db");
   574         -	if (snprintf_ret >= sizeof(dbfilename)) {
   575         -		fprintf(stderr, "Unable to set database filename.  Aborting.\n");
   576         -
   577         -		return(1);
          628  +	if (argc == 3 && strcmp(argv[1], "-sqlite3") == 0) {
          629  +		return(appfs_sqlite3(argv[2]));
   578    630   	}
   579    631   
   580    632   	return(fuse_main(argc, argv, &appfs_oper, NULL));
   581    633   }
   582    634    

Modified appfsd.tcl from [1a0526f9cf] to [e0ad1bdba0].

    76     76   		if {![regexp {^[0-9a-f]*$} $value]} {
    77     77   			return false
    78     78   		}
    79     79   
    80     80   		return true
    81     81   	}
    82     82   
    83         -	proc _db {args} {
    84         -		return [uplevel 1 [list ::appfs::db {*}$args]]
    85         -	}
    86         -
    87     83   	proc _normalizeOS {os} {
    88     84   		set os [string tolower [string trim $os]]
    89     85   
    90     86   		switch -- $os {
    91     87   			"linux" - "freebsd" - "openbsd" - "netbsd" {
    92     88   				return $os
    93     89   			}
................................................................................
   120    116   		return -code error "Unable to normalize CPU: $cpu"
   121    117   	}
   122    118   
   123    119   	proc init {} {
   124    120   		if {[info exists ::appfs::init_called]} {
   125    121   			return
   126    122   		}
          123  +
          124  +		# Force [parray] to be loaded
          125  +		catch {
          126  +			parray does_not_exist
          127  +		}
   127    128   
   128    129   		set ::appfs::init_called 1
   129    130   
   130    131   		if {![info exists ::appfs::db]} {
   131    132   			file mkdir $::appfs::cachedir
   132    133   
   133    134   			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]
   134    135   		}
   135    136   
   136    137   		# Create tables
   137         -		_db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate, ttl);}
   138         -		_db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest);}
   139         -		_db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory);}
          138  +		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate, ttl);}
          139  +		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest);}
          140  +		db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory);}
   140    141   
   141    142   		# Create indexes
   142         -		_db eval {CREATE INDEX IF NOT EXISTS sites_index ON sites (hostname);}
   143         -		_db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, package, version, os, cpuArch);}
   144         -		_db eval {CREATE INDEX IF NOT EXISTS files_index ON files (package_sha1, file_name, file_directory);}
          143  +		db eval {CREATE INDEX IF NOT EXISTS sites_index ON sites (hostname);}
          144  +		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, package, version, os, cpuArch);}
          145  +		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (package_sha1, file_name, file_directory);}
   145    146   	}
   146    147   
   147    148   	proc download {hostname hash {method sha1}} {
   148    149   		set url "http://$hostname/appfs/$method/$hash"
   149    150   		set file [_cachefile $url $hash]
   150    151   
   151    152   		if {![file exists $file]} {
................................................................................
   154    155   
   155    156   		return $file
   156    157   	}
   157    158   
   158    159   	proc getindex {hostname} {
   159    160   		set now [clock seconds]
   160    161   
   161         -		set lastUpdates [_db eval {SELECT lastUpdate, ttl FROM sites WHERE hostname = $hostname LIMIT 1;}]
          162  +		set lastUpdates [db eval {SELECT lastUpdate, ttl FROM sites WHERE hostname = $hostname LIMIT 1;}]
   162    163   		if {[llength $lastUpdates] == 0} {
   163    164   			set lastUpdate 0
   164    165   			set ttl 0
   165    166   		} else {
   166    167   			set lastUpdate [lindex $lastUpdates 0]
   167    168   			set ttl [lindex $lastUpdates 1]
   168    169   		}
................................................................................
   184    185   			}
   185    186   			::http::reset $token
   186    187   			::http::cleanup $token
   187    188   		}
   188    189   
   189    190   		if {![info exists indexhash_data]} {
   190    191   			# Cache this result for 60 seconds
   191         -			_db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::nttl);}
          192  +			db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::nttl);}
   192    193   
   193    194   			return -code error "Unable to fetch $url"
   194    195   		}
   195    196   
   196    197   		set indexhash [lindex [split $indexhash_data ","] 0]
   197    198   
   198    199   		if {![_isHash $indexhash]} {
................................................................................
   234    235   			if {![_isHash $pkgInfo(hash)]} {
   235    236   				continue
   236    237   			}
   237    238   
   238    239   			lappend curr_packages $pkgInfo(hash)
   239    240   
   240    241   			# Do not do any additional work if we already have this package
   241         -			set existing_packages [_db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
          242  +			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
   242    243   			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
   243    244   				continue
   244    245   			}
   245    246   
   246    247   			if {$pkgInfo(isLatest)} {
   247         -				_db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo($package) AND os = $pkgInfo($package) AND cpuArch = $pkgInfo(cpuArch);}
          248  +				db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo($package) AND os = $pkgInfo($package) AND cpuArch = $pkgInfo(cpuArch);}
   248    249   			}
   249    250   
   250         -			_db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
          251  +			db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
   251    252   		}
   252    253   
   253    254   		# Look for packages that have been deleted
   254         -		set found_packages [_db eval {SELECT sha1 FROM packages WHERE hostname = $hostname;}]
          255  +		set found_packages [db eval {SELECT sha1 FROM packages WHERE hostname = $hostname;}]
   255    256   		foreach package $found_packages {
   256    257   			set found_packages_arr($package) 1
   257    258   		}
   258    259   
   259    260   		foreach package $curr_packages {
   260    261   			unset -nocomplain found_packages_arr($package)
   261    262   		}
   262    263   
   263    264   		foreach package [array names found_packages_arr] {
   264         -			_db eval {DELETE FROM packages WHERE hostname = $hostname AND sha1 = $package;}
          265  +			db eval {DELETE FROM packages WHERE hostname = $hostname AND sha1 = $package;}
   265    266   		}
   266    267   
   267         -		_db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::ttl);}
          268  +		db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::ttl);}
   268    269   
   269    270   		return COMPLETE
   270    271   	}
   271    272   
   272    273   	proc getpkgmanifest {hostname package_sha1} {
   273         -		set haveManifests [_db eval {SELECT haveManifest FROM packages WHERE sha1 = $package_sha1 LIMIT 1;}]
          274  +		set haveManifests [db eval {SELECT haveManifest FROM packages WHERE sha1 = $package_sha1 LIMIT 1;}]
   274    275   		set haveManifest [lindex $haveManifests 0]
   275    276   
   276    277   		if {$haveManifest} {
   277    278   			return COMPLETE
   278    279   		}
   279    280   
   280    281   		if {![_isHash $package_sha1]} {
................................................................................
   282    283   		}
   283    284   
   284    285   		set file [download $hostname $package_sha1]
   285    286   		set fd [open $file]
   286    287   		set pkgdata [read $fd]
   287    288   		close $fd
   288    289   
   289         -		_db transaction {
          290  +		db transaction {
   290    291   			foreach line [split $pkgdata "\n"] {
   291    292   				set line [string trim $line]
   292    293   
   293    294   				if {$line == ""} {
   294    295   					continue
   295    296   				}
   296    297   
................................................................................
   316    317   				}
   317    318   
   318    319   				set fileInfo(name) [join $work ","]
   319    320   				set fileInfo(name) [split [string trim $fileInfo(name) "/"] "/"]
   320    321   				set fileInfo(directory) [join [lrange $fileInfo(name) 0 end-1] "/"]
   321    322   				set fileInfo(name) [lindex $fileInfo(name) end]
   322    323   
   323         -				_db eval {INSERT INTO files (package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory) VALUES ($package_sha1, $fileInfo(type), $fileInfo(time), $fileInfo(source), $fileInfo(size), $fileInfo(perms), $fileInfo(sha1), $fileInfo(name), $fileInfo(directory) );}
   324         -				_db eval {UPDATE packages SET haveManifest = 1 WHERE sha1 = $package_sha1;}
          324  +				db eval {INSERT INTO files (package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory) VALUES ($package_sha1, $fileInfo(type), $fileInfo(time), $fileInfo(source), $fileInfo(size), $fileInfo(perms), $fileInfo(sha1), $fileInfo(name), $fileInfo(directory) );}
          325  +				db eval {UPDATE packages SET haveManifest = 1 WHERE sha1 = $package_sha1;}
   325    326   			}
   326    327   		}
   327    328   
   328    329   		return COMPLETE
   329    330   	}
   330    331   }