AppFS
Check-in [48a551bede]
Login
HomeTimelineFilesBranchesTagsTicketsWiki
Overview
Comment:Updated to populate SQLite database with all package data
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 48a551bede7cc48fce7871f033ad1dd89e5ead6b
User & Date: rkeene on 2014-09-08 02:33:28
Other Links: manifest | tags
Context
2014-09-08
02:38
Separated manifest fetching from index fetching check-in: b5d1616f0f user: rkeene tags: trunk
02:33
Updated to populate SQLite database with all package data check-in: 48a551bede user: rkeene tags: trunk
01:24
More work on basics check-in: a189f64907 user: rkeene tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified README.md from [5b04e66b64] to [15ef741d8b].

︙
23
24
25
26
27
28
29
30
31



23
24
25
26
27
28
29


30
31








-
-
+
+

    		type == file; extraData = size,sha1

    /opt/appfs/hostname/{sha1,package/os-cpuArch/version}/file
    	Fetches: http://hostname/appfs/sha1/<sha1>

Database
--------
    hostname_to_packages(hostname, sha1, package, version, os, cpuArch, isLatest)
    package_to_files(package_sha1, type, time, source, size, file_sha1, name)
    packages(hostname, sha1, package, version, os, cpuArch, isLatest)
    files(package_sha1, type, time, source, size, file_sha1, file_name, file_directory)

Modified appfs.c from [1eae70cf2b] to [4b3327c544]. 

1
2
3

4
5
6
7
8
9
10

1
2
3
4
5
6
7
8
9
10
11




+








#define FUSE_USE_VERSION 26

#include <string.h>
#include <stdarg.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <fuse.h>
#include <tcl.h>

#define APPFS_DEBUG(x...) { fprintf(stderr, "%i:%s: ", __LINE__, __func__); fprintf(stderr, x); fprintf(stderr, "\n"); }
︙
110
111
112
113
114
115
116
117
118
119
120

























121
122
123
124
125
126
127

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

111
112
113
114
115
116
117
118



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

150



151
152
153
154
155
156


157
158
159
160
161
162
163









-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+






-
+
-
-
-






-
-








			return("arm");
		case APPFS_CPU_UNKNOWN:
			return("unknown");
	}

	return("unknown");
}

static struct appfs_package *appfs_getindex(const char *hostname, int *package_count_p) {
	Tcl_Obj *objv[2];
	Tcl_Obj *packages_tcl;
static int appfs_Tcl_Eval(Tcl_Interp *interp, int objc, const char *cmd, ...) {
	Tcl_Obj **objv;
	const char *arg;
	va_list argp;
	int retval;
	int i;

	objv = ckalloc(sizeof(*objv) * objc);
	objv[0] = Tcl_NewStringObj(cmd, -1);

	va_start(argp, cmd);
	for (i = 1; i < objc; i++) {
		arg = va_arg(argp, const char *);
		objv[i] = Tcl_NewStringObj(arg, -1);
	}
	va_end(argp);

	retval = Tcl_EvalObjv(interp, objc, objv, 0);

	ckfree(objv);

	return(retval);
}

static struct appfs_package *appfs_getindex(const char *hostname, int *package_count_p) {
	int tcl_ret;

	if (package_count_p == NULL) {
		return(NULL);
	}

	objv[0] = Tcl_NewStringObj("::appfs::getindex", -1);
	tcl_ret = appfs_Tcl_Eval(interp, 2, "::appfs::getindex", hostname);
	objv[1] = Tcl_NewStringObj(hostname, -1);

	tcl_ret = Tcl_EvalObjv(interp, 2, objv, 0);
	if (tcl_ret != TCL_OK) {
		APPFS_DEBUG("Call to ::appfs::getindex failed: %s", Tcl_GetStringResult(interp));

		return(NULL);
	}

	packages_tcl = Tcl_GetObjResult(interp);

	return(NULL);
}

static int appfs_getfile(const char *hostname, const char *sha1) {
}

static int appfs_getmanifest(const char *hostname, const char *sha1) {
︙
215
216
217
218
219
220
221
222









223
224
225
226
227
228
229
230
231
232
233

233
234
235
236
237
238
239

240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259








-
+
+
+
+
+
+
+
+
+












		return(1);
	}

	tcl_ret = Tcl_Eval(interp, ""
#include "appfs.tcl.h"
	"");
	if (tcl_ret != TCL_OK) {
		fprintf(stderr, "Unable to initialize Tcl AppFS Script.  Aborting.\n");
		fprintf(stderr, "Unable to initialize Tcl AppFS script.  Aborting.\n");

		return(1);
	}

	tcl_ret = appfs_Tcl_Eval(interp, 1, "::appfs::init");
	if (tcl_ret != TCL_OK) {
		fprintf(stderr, "Unable to initialize Tcl AppFS script (::appfs::init).  Aborting.\n");
		fprintf(stderr, "Tcl Error is: %s\n", Tcl_GetStringResult(interp));

		return(1);
	}

#ifdef APPFS_TEST_DRIVER
	return(appfs_test_driver());
#else
	return(fuse_main(argc, argv, &appfs_oper, NULL));
#endif
}

Modified appfs.tcl from [07f0ced01b] to [58a43bb4f1]. 

1
2
3
4
5
6
7
8
9
10
11
12
13
14

1
2
3
4
5
6

7
8
9
10
11
12
13







-








#! /usr/bin/env tclsh

package require http 2.7
package require sqlite3

namespace eval ::appfs {
	variable sites [list]
	variable cachedir "/tmp/appfs-cache"

	proc _hash_sep {hash {seps 4}} {
		for {set idx 0} {$idx < $seps} {incr idx} {
			append retval "[string range $hash [expr {$idx * 2}] [expr {($idx * 2) + 1}]]/"
		}
		append retval "[string range $hash [expr {$idx * 2}] end]"
︙
40
41
42
43
44
45
46





















47
48
49
50
51
52
53

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








			} else {
				file delete -force -- $tmpfile
			}
		}

		return $file
	}

	proc _db {args} {
		return [uplevel 1 [list ::appfs::db {*}$args]]
	}

	proc init {} {
		if {[info exists ::appfs::init_called]} {
			return
		}

		set ::appfs::init_called 1

		if {![info exists ::appfs::db]} {
			file mkdir $::appfs::cachedir

			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]
		}

		_db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest);}
		_db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, file_sha1, file_name, file_directory);}
	}

	proc getindex {hostname} {
		if {[string match "*\[/~\]*" $hostname]} {
			return -code error "Invalid hostname"
		}

		set url "http://$hostname/appfs/index"
︙
96
97
98
99
100
101
102
103
104
105
























































106
107
108
109
110
111
112
113
114
115
116
117
118

116
117
118
119
120
121
122



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191








-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+














			}

			if {![regexp {^[0-9a-f]*$} $pkgInfo(hash)]} {
				continue
			}

			set packages($pkgInfo(package)) [array get pkgInfo]
		}

		return [array get packages]

			# Do not do any additional work if we already have this package
			set existing_packages [_db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
				continue
			}

			if {$pkgInfo(isLatest)} {
				_db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo($package) AND os = $pkgInfo($package) AND cpuArch = $pkgInfo(cpuArch);}
			}

			_db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest) );}

			set file [download $hostname $pkgInfo(hash)]
			set fd [open $file]
			set pkgdata [read $fd]
			close $fd

			foreach line [split $pkgdata "\n"] {
				set line [string trim $line]

				if {[string match "*/*" $line]} {
					continue
				}

				if {$line == ""} {
					continue
				}

				set work [split $line ","]

				unset -nocomplain fileInfo
				set fileInfo(type) [lindex $work 0]
				set fileInfo(time) [lindex $work 1]
				set fileInfo(name) [lindex $work end]

				set fileInfo(name) [split [string trim $fileInfo(name) "/"] "/"]
				set fileInfo(directory) [join [lrange $fileInfo(name) 0 end-1] "/"]
				set fileInfo(name) [lindex $fileInfo(name) end]

				set work [lrange $work 2 end-1]
				switch -- $fileInfo(type) {
					"file" {
						set fileInfo(size) [lindex $work 0]
						set fileInfo(sha1) [lindex $work 1]
					}
					"symlink" {
						set fileInfo(source) [lindex $work 0]
					}
				}

				_db eval {INSERT INTO files (package_sha1, type, time, source, size, file_sha1, file_name, file_directory) VALUES ($pkgInfo(hash), $fileInfo(type), $fileInfo(time), $fileInfo(source), $fileInfo(size), $fileInfo(sha1), $fileInfo(name), $fileInfo(directory) );}
			}
		}

		return COMPLETE
	}

	proc download {hostname hash {method sha1}} {
		set url "http://$hostname/appfs/$method/$hash"
		set file [_cachefile $url $hash]

		if {![file exists $file]} {
			return -code error "Unable to fetch"
		}

		return $file
	}
}