M README +4 -4
@@ 28,16 28,16 @@ If you'd rather install from source, you
Once these are installed:
- $ tar -zxvf thingfish-0.1.0.tgz
- $ cd thingfish-0.1.0
+ $ tar -zxvf thingfish-0.X.N.tgz
+ $ cd thingfish-0.X.N
$ su -
# rake install
If you want to help out with development, run tests, or generate documentation,
you'll need a few more things:
- * RSpec (>= 1.0.5): http://rspec.rubyforge.org/
- * rcov (>= 0.7.0): http://eigenclass.org/hiki.rb?rcov
+ * RSpec (>= 1.1.11): http://rspec.rubyforge.org/
+ * rcov (>= 0.8.1): http://eigenclass.org/hiki.rb?rcov
* lockfile (>= 1.4.3): http://codeforpeople.com/lib/ruby/lockfile/
* rcodetools (>= 0.7.0): http://eigenclass.org/hiki/rcodetools
* Redcloth (>= 3.0.4): http://whytheluckystiff.net/ruby/redcloth/
M Rakefile +1 -1
@@ 151,7 151,7 @@ end
### This assumes exuberant ctags, since ctags 'native' doesn't support ruby anyway.
desc "Generate a ctags 'tags' file from ThingFish source"
task :ctags do
- run %w{ ctags -R lib plugins misc ext }
+ run %w{ ctags -R lib plugins misc }
end
M docs/manual/src/Developers_Guide/connecting.page +8 -2
@@ 93,10 93,16 @@ Returns a list of URIs for files which m
<dl class="apidocs kvlist">
<dt>GET /search?«querystring»</dt>
<dd>Find files matching the criteria in the request's query string. The query string uses a
- (somewhat) simple convention to build the logic of the search:</dd>
+ simple convention to build the logic of the search:</dd>
<dd class="example">
- <p>(to be documented after the metastore changes are done)</p>
+ <pre><code>
+ /search?format=image/jpeg&title=*metric*&created=*Nov%205*
+ </code></pre>
+ </dd>
+ <dd>
+ In this example, we search for all jpeg images with the phrase 'metric' in their title,
+ that were uploaded on November 5th.
</dd>
<dt>POST /search</dt>
M docs/manual/src/getting-started.page +87 -62
@@ 55,8 55,8 @@ If you're installing from source, you'll
Once you have those installed, you can install ThingFish like so:
<pre>
- $ <kbd>tar -zxvf thingfish-0.1.0.tgz</kbd>
- $ <kbd>cd thingfish-0.1.0</kbd>
+ $ <kbd>tar -zxvf thingfish-0.X.N.tgz</kbd>
+ $ <kbd>cd thingfish-0.X.N</kbd>
$ <kbd>su -</kbd>
# <kbd>rake install</kbd>
</pre>
@@ 64,11 64,11 @@ Once you have those installed, you can i
The dependencies above are enough to run the server, but if you want to help out with development,
run tests, or generate documentation, you'll need a few more things:
+* RSpec (>= 1.1.11): "http://rspec.rubyforge.org/":http://rspec.rubyforge.org/
+* rcov (>= 0.8.1): "http://eigenclass.org/hiki.rb?rcov":http://eigenclass.org/hiki.rb?rcov
* lockfile (>= 1.4.3): "http://codeforpeople.com/lib/ruby/lockfile/":http://codeforpeople.com/lib/ruby/lockfile/
* rcodetools (>= 0.7.0): "http://eigenclass.org/hiki/rcodetools":http://eigenclass.org/hiki/rcodetools
-* rcov (>= 0.7.0): "http://eigenclass.org/hiki.rb?rcov":http://eigenclass.org/hiki.rb?rcov
* Redcloth (>= 3.0.4): "http://whytheluckystiff.net/ruby/redcloth/":http://whytheluckystiff.net/ruby/redcloth/
-* RSpec (>= 1.1.1): "http://rspec.rubyforge.org/":http://rspec.rubyforge.org/
* ultraviolet (>= 0.10.2): "http://ultraviolet.rubyforge.org/":http://ultraviolet.rubyforge.org/
If you have RubyGems installed, you can install these automatically via the
@@ 98,7 98,7 @@ file somewhere else with the @-f@ flag:
h3. Configuration File
-The config file is a YAML file, and chooses sane values as defaults. An absolutely
+The config file is a YAML file, and chooses sane values as defaults. A
minimalistic configuration should look something like this:
<?example { language: yaml, caption: Example minimalistic config file } ?>
@@ 118,28 118,35 @@ Here's an example of a fairly full-featu
---
port: 80
ip: 127.0.0.1
-spooldir: /home/thingfish/var/spool
+datadir: /home/thingfish
+spooldir: /var/tmp
+resource_dir: var/www
bufsize: 16384
user: daemon
daemon: true
-pidfile: /home/thingfish/var/thingfish.pid
+pidfile: thingfish.pid
+connection_timeout: 15
+pipeline_max: 50
+memory_bodysize_max: 256000
-defaulthandler:
- html_index: index.rhtml
- resource_dir: /home/thingfish/var/www
+profiling:
+ enabled: true
logging:
level: info
- logfile: /home/thingfish/var/thingfish.log
+ logfile: thingfish.log
plugins:
filestore:
name: filesystem
maxsize: 1073741824
- root: /home/thingfish/var
metastore:
- name: sequel
- sequel_connect: postgres://thingfish@localhost/tf
+ name: marshalled
+ urimap:
+ /inspect: inspect
+ /upload: formupload
+ /metadata: simplemetadata
+ /search: simplesearch
filters:
- html
- xml
@@ 147,23 154,6 @@ plugins:
- json
- mp3info
- exif
- handlers:
- - inspect:
- uris: /inspect
- resource_dir: /home/thingfish/plugins/thingfish-inspecthandler/resources
- - formupload:
- uris: /upload
- resource_dir: /home/thingfish/plugins/thingfish-formuploadhandler/resources
- - simplemetadata:
- uris: /metadata
- resource_dir: var/www
- - simplesearch:
- uris: /search
- resource_dir: var/www
- - status:
- uris: /status
- stat_uris: [/, /inspect, /upload, /metadata, /status]
- resource_dir: /home/thingfish/plugins/thingfish-statushandler/resources
<?end?>
@@ 173,29 163,63 @@ The meaning of the config values are as
<dt>port</dt>
<dd class="description">The port the daemon should listen on.</dd>
<dd class="default">Defaults to <code>3474</code></dd>
-
+
<dt>ip</dt>
<dd class="description">The ip the daemon should bind to.</dd>
<dd class="default">Defaults to <code>0.0.0.0</code></dd>
- <dd class="note"><strong>NOTE:</strong> This setting is currently unused, as
- Ebb is hard-coded to bind to INADDR_ANY (0.0.0.0). We hope to have a patch
- submitted that changes this soon, but for the time-being, we're just leaving
- it in the config in the hopes it'll be useful eventually.</dd>
-
+
<dt>user</dt>
<dd class="description">The username to run as if we're started as root.</dd>
<dd class="default">No default</dd>
-
+
+ <dt>datadir</dt>
+ <dd class="description">The location on disk that ThingFish uses to store data files,
+ spoolfiles, its pid, and anything.</dd>
+ <dd class="default">Defaults to <code>System_Temp_Dir/thingfish</code></dd>
+
+ <dt>spooldir</dt>
+ <dd class="description">The location on disk that ThingFish stores temporary files to.
+ If set to a relative path, this directory is created under the current
+ <strong>datadir</strong>.</dd>
+ <dd class="default">Defaults to <code>spool</code> under the <strong>datadir</strong>.</dd>
+
+ <dt>resource_dir</dt>
+ <dd class="description">The default location for handlers and filters to find
+ files that maybe be required for their use.
+ <dd class="default">Defaults to a directory called
+ <code>thingfish/default</code> under Ruby's shared datadir.</dd>
+
+ <dt>bufsize</dt>
+ <dd class="description">During file transfers, this option (in bytes)
+ controls how much data to buffer to (or from) the client at once.</dd>
+ <dd class="default">Defaults to <code>16384</code>, or 16k.</dd>
+
<dt>daemon</dt>
<dd class="description">Either <code>true</code> or <code>false</code>.
Whether or not to detach from the controlling terminal.</dd>
<dd class="default">Defaults to <code>false</code></dd>
-
+
<dt>pidfile</dt>
<dd class="description">A filename to write the daemon's pid to. Ignored
if daemon mode is not enabled.</dd>
<dd class="default">No default</dd>
-
+
+ <dt>connection_timeout</dt>
+ <dd class="description">If no activity is seen on a client socket for this
+ amount of time (in seconds), the connection is closed.</dd>
+ <dd class="default">Defaults to <code>30</code> seconds.</dd>
+
+ <dt>pipeline_max</dt>
+ <dd class="description">If a client requests HTTP pipelining via the
+ <code>Keep-Alive</code> header, this limits how many requests can be made
+ within a single connection.</dd>
+ <dd class="default">Defaults to <code>100</code> requests.</dd>
+
+ <dt>memory_bodysize_max</dt>
+ <dd class="description">When uploading, keep files under this size (in bytes) in memory
+ instead of spooling them to tempfilesp</dd>
+ <dd class="default">Defaults to <code>100k</code>.</dd>
+
<dt>defaulthandler</dt>
<dd class="description">This subsection contains configuration values for the
top-level handler object of the daemon.</dd>
@@ 210,8 234,7 @@ The meaning of the config values are as
<dt>resource_dir</dt>
<dd class="description">The directory to search for static resources
relative to the current directory,
- <dd class="default">Defaults to a directory called
- <code>thingfish/default</code> under Ruby's shared datadir </dd>
+ <dd class="default">Defaults to the global @resource_dir@ setting.</dd>
</dl>
</dd>
@@ 294,17 317,16 @@ The meaning of the config values are as
</dl>
</dd>
- <dt>handlers</dt>
- <dd class="description">Configuration of any handler plugins you wish to
- be loaded to handle various part of the urispace of the server. This
- is in the form of a YAML array of hashes, one plugin per hash. Each
- hash has the name of the plugin to be loaded as the key, and any
- plugin-specific configuration is stored in the value. This
- plugin-specific configuration must contain at least a uri mount
- point. See the documentation for the plugin itself for additional
- values that can be set. For more information about writing handlers,
- see the "Handler":../03.Hackers_Guide/writing-handlers.html
- documentation.
+ <dt>urimap</dt>
+ <dd class="description">
+ Configuration of any handler plugins you wish to be loaded to
+ handle various part of the urispace of the server. This is in the
+ form of a YAML hash, one uri per hash key. Each value has an array
+ of plugins that should respond to that particular uri, and the
+ plugin's options (if any.) See the documentation for the plugin
+ itself for additional values that can be set. For more information
+ about writing handlers, see the
+ "Handler":../03.Hackers_Guide/writing-handlers.html documentation.
</dd>
<dd class="subsection">
<dl>
@@ 313,23 335,26 @@ The meaning of the config values are as
<p>The following are all valid methods for defining URI mount
points for a handler:</p>
<?example { language: yaml, caption: As the only value } ?>
-- handlername: /mount
+/mount: handlername
<?end?>
-<?example { language: yaml, caption: As a string value in the uris pair of the options hash } ?>
-- handlername:
- uris: /mount
+<?example { language: yaml, caption: Multiple handlers for a specific mount point } ?>
+/mount:
+ - handlername
+ - anotherhandlername
<?end?>
-<?example { language: yaml, caption: As an array of URIs in the uris pair of the options hash } ?>
-- handlername:
- uris: [/mount, /mount2]
+<?example { language: yaml, caption: A handler with custom options } ?>
+/mount:
+ - handlername:
+ option: value
+ another: value
<?end?>
</dd>
</dl>
</dd>
- <dd class="default">The default is to only handle requests via the default
- handler.</dd>
+ <dd class="default">The default behaviour is to only handle requests via the
+ default handler.</dd>
</dl>
</dd>
M etc/thingfish.conf +14 -15
@@ 17,20 17,19 @@
# - HTML filter (browsable server)
#
---
-defaulthandler:
- resource_dir: var/www
+resource_dir: var/www
plugins:
- handlers:
- - formupload:
- uris: /upload
- resource_dir: plugins/thingfish-handler-formupload/resources
- - simplemetadata:
- uris: /metadata
- resource_dir: var/www
- - simplesearch:
- uris: /search
- resource_dir: var/www
- filters:
- - html:
- resource_dir: var/www
+ urimap:
+ /upload:
+ - formupload:
+ resource_dir: plugins/thingfish-handler-formupload/resources
+ /metadata:
+ - simplemetadata:
+ resource_dir: var/www
+ /search:
+ - simplesearch:
+ resource_dir: var/www
+ filters:
+ - html:
+ resource_dir: var/www
M etc/thingfish.conf.advanced +15 -17
@@ 8,32 8,32 @@
# - Bind only to the localhost interface, port 80.
# - Become a daemon.
# - Store metadata and filedata persistently.
-# - Try and load profiling.
+# - Try and load the built in profiler.
# - All data is stored under /home/thingfish, except tempfiles, which are created in /var/tmp.
-# - Install some useful intropection and file upload handlers.
+# - Install some basic handlers.
# - Install some basic response filters.
# - Install some basic extraction filters.
+# - Drop connections that idle for longer than 15 seconds
+# - Allow pipelined requests, with a maximum of 50 at a time
+# - Keep body data under 250k in memory, instead of creating a tempfile
#
-
-# NOTE: The 'ip' setting is currently unused, as Ebb is hard-coded to bind to INADDR_ANY (0.0.0.0).
-# We hope to have a patch submitted that changes this soon, but for the time-being, we're just leaving
-# it in the config in the hopes it'll be useful eventually.
---
port: 80
ip: 127.0.0.1
+user: daemon
datadir: /home/thingfish
spooldir: /var/tmp
+resource_dir: var/www
bufsize: 16384
-user: daemon
daemon: true
pidfile: thingfish.pid
+connection_timeout: 15
+pipeline_max: 50
+memory_bodysize_max: 256000
profiling:
enabled: true
-defaulthandler:
- html_index: index.rhtml
-
logging:
level: info
logfile: thingfish.log
@@ 44,6 44,11 @@ plugins:
maxsize: 1073741824
metastore:
name: marshalled
+ urimap:
+ /inspect: inspect
+ /upload: formupload
+ /metadata: simplemetadata
+ /search: simplesearch
filters:
- html
- xml
@@ 51,11 56,4 @@ plugins:
- json
- mp3info
- exif
- handlers:
- - inspect:
- uris: /inspect
- - formupload:
- uris: [ /upload ]
- - metadata:
- uris: /metadata
R etc/thingfish.conf.example => +0 -36
@@ 1,36 0,0 @@
-#
-# This is a simple configuration file for ThingFish that will run
-# from the distribution directory.
-#
-# If you've just extracted ThingFish from a tarball, you can use the
-# included 'run' script to start experimenting at
-# http://localhost:3474/
-#
-# % ./run
-#
-# Behaviors:
-# - Bind to all interfaces, port 3474
-# - Stay in the foreground
-# - Only log warnings and errors
-# - Store all data (metastore and filestore) in RAM
-# - Only the minimally-functional handlers are installed.
-# - HTML filter (browsable server)
-#
----
-defaulthandler:
- resource_dir: var/www
-
-plugins:
- handlers:
- - formupload:
- uris: /upload
- resource_dir: plugins/thingfish-handler-formupload/resources
- - simplemetadata:
- uris: /metadata
- resource_dir: var/www
- - simplesearch:
- uris: /search
- resource_dir: var/www
- filters:
- - html:
- resource_dir: var/www
R => +0 -275
@@ 1,275 0,0 @@
This patch is intended for ApacheBench version 2.3, which is distributed
with Apache 2.2.9.
--- ab.c.orig 2008-06-19 21:59:04.000000000 -0700
+++ ab.c 2008-06-19 22:44:25.000000000 -0700
@@ -84,6 +84,10 @@
** Version 2.3
** SIGINT now triggers output_results().
** Contributed by colm, March 30, 2006
+ **
+ ** Version 2.3.1
+ ** PUT and DELETE HTTP method support.
+ ** Contributed by Mahlon E. Smith <mahlon@martini.nu>, June 2008.
**/
/* Note: this version string should start with \d+[\d\.]* and be a valid
@@ -95,7 +99,7 @@
* ab - or to due to a change in the distribution it is compiled with
* (such as an APR change in for example blocking).
*/
-#define AP_AB_BASEREVISION "2.3"
+#define AP_AB_BASEREVISION "2.3.1"
/*
* BUGS:
@@ -261,7 +265,7 @@
int verbosity = 0; /* no verbosity by default */
int recverrok = 0; /* ok to proceed after socket receive errors */
-int posting = 0; /* GET by default */
+int method = 0; /* -2 => DELETE, -1 => HEAD, 0 => GET (default), 1 => POST, 2 => PUT */
int requests = 1; /* Number of requests to make */
int heartbeatres = 100; /* How often do we say we're alive */
int concurrency = 1; /* Number of multiple requests to make */
@@ -631,7 +635,7 @@
c->connect = tnow;
c->rwrote = 0;
c->rwrite = reqlen;
- if (posting)
+ if (method)
c->rwrite += postlen;
}
else if (tnow > c->connect + aprtimeout) {
@@ -757,8 +761,10 @@
if (keepalive)
printf("Keep-Alive requests: %d\n", doneka);
printf("Total transferred: %" APR_INT64_T_FMT " bytes\n", totalread);
- if (posting > 0)
+ if (method == 1)
printf("Total POSTed: %" APR_INT64_T_FMT "\n", totalposted);
+ if (method == 2)
+ printf("Total PUT: %" APR_INT64_T_FMT "\n", totalposted);
printf("HTML transferred: %" APR_INT64_T_FMT " bytes\n", totalbread);
/* avoid divide by zero */
@@ -771,7 +777,7 @@
(double) timetaken * 1000 / done);
printf("Transfer rate: %.2f [Kbytes/sec] received\n",
(double) totalread / 1024 / timetaken);
- if (posting > 0) {
+ if (method > 0) {
printf(" %.2f kb/s sent\n",
(double) totalposted / timetaken / 1024);
printf(" %.2f kb/s total\n",
@@ -1042,10 +1048,14 @@
printf("<tr %s><th colspan=2 %s>Total transferred:</th>"
"<td colspan=2 %s>%" APR_INT64_T_FMT " bytes</td></tr>\n",
trstring, tdstring, tdstring, totalread);
- if (posting > 0)
+ if (method == 1)
printf("<tr %s><th colspan=2 %s>Total POSTed:</th>"
"<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
trstring, tdstring, tdstring, totalposted);
+ if (method == 2)
+ printf("<tr %s><th colspan=2 %s>Total PUT:</th>"
+ "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
+ trstring, tdstring, tdstring, totalposted);
printf("<tr %s><th colspan=2 %s>HTML transferred:</th>"
"<td colspan=2 %s>%" APR_INT64_T_FMT " bytes</td></tr>\n",
trstring, tdstring, tdstring, totalbread);
@@ -1058,7 +1068,7 @@
printf("<tr %s><th colspan=2 %s>Transfer rate:</th>"
"<td colspan=2 %s>%.2f kb/s received</td></tr>\n",
trstring, tdstring, tdstring, (double) totalread / timetaken);
- if (posting > 0) {
+ if (method > 0) {
printf("<tr %s><td colspan=2 %s> </td>"
"<td colspan=2 %s>%.2f kb/s sent</td></tr>\n",
trstring, tdstring, tdstring,
@@ -1466,8 +1476,8 @@
cl = strstr(c->cbuff, "Content-length:");
if (cl) {
c->keepalive = 1;
- /* response to HEAD doesn't have entity body */
- c->length = posting >= 0 ? atoi(cl + 16) : 0;
+ /* response to HEAD and DELETE doesn't have entity body */
+ c->length = method >= 0 ? atoi(cl + 16) : 0;
}
/* The response may not have a Content-Length header */
if (!cl) {
@@ -1532,6 +1542,7 @@
int i;
apr_status_t status;
int snprintf_res = 0;
+ char *verb = "GET";
#ifdef NOT_ASCII
apr_size_t inbytes_left, outbytes_left;
#endif
@@ -1588,24 +1599,44 @@
}
/* setup request */
- if (posting <= 0) {
+ if (method <= 0) {
+ switch (method) {
+ case -2:
+ verb = "DELETE";
+ break;
+ case -1:
+ verb = "HEAD";
+ break;
+ case 0:
+ verb = "GET";
+ break;
+ }
snprintf_res = apr_snprintf(request, sizeof(_request),
"%s %s HTTP/1.0\r\n"
"%s" "%s" "%s"
"%s" "\r\n",
- (posting == 0) ? "GET" : "HEAD",
+ verb,
(isproxy) ? fullurl : path,
keepalive ? "Connection: Keep-Alive\r\n" : "",
cookie, auth, hdrs);
}
else {
+ switch (method) {
+ case 1:
+ verb = "POST";
+ break;
+ case 2:
+ verb = "PUT";
+ break;
+ }
snprintf_res = apr_snprintf(request, sizeof(_request),
- "POST %s HTTP/1.0\r\n"
+ "%s %s HTTP/1.0\r\n"
"%s" "%s" "%s"
"Content-length: %" APR_SIZE_T_FMT "\r\n"
"Content-type: %s\r\n"
"%s"
"\r\n",
+ verb,
(isproxy) ? fullurl : path,
keepalive ? "Connection: Keep-Alive\r\n" : "",
cookie, auth,
@@ -1622,9 +1653,9 @@
reqlen = strlen(request);
/*
- * Combine headers and (optional) post file into one contineous buffer
+ * Combine headers and (optional) post file into one continuous buffer
*/
- if (posting == 1) {
+ if (method >= 1) {
char *buff = malloc(postlen + reqlen + 1);
if (!buff) {
fprintf(stderr, "error creating request buffer: out of memory\n");
@@ -1825,12 +1856,14 @@
fprintf(stderr, " -t timelimit Seconds to max. wait for responses\n");
fprintf(stderr, " -b windowsize Size of TCP send/receive buffer, in bytes\n");
fprintf(stderr, " -p postfile File containing data to POST. Remember also to set -T\n");
+ fprintf(stderr, " -u putfile File containing data to PUT.\n");
fprintf(stderr, " -T content-type Content-type header for POSTing, eg.\n");
fprintf(stderr, " 'application/x-www-form-urlencoded'\n");
fprintf(stderr, " Default is 'text/plain'\n");
fprintf(stderr, " -v verbosity How much troubleshooting info to print\n");
fprintf(stderr, " -w Print out results in HTML tables\n");
- fprintf(stderr, " -i Use HEAD instead of GET\n");
+ fprintf(stderr, " -i Send a HEAD request\n");
+ fprintf(stderr, " -D Send a DELETE request\n");
fprintf(stderr, " -x attributes String to insert as table attributes\n");
fprintf(stderr, " -y attributes String to insert as tr attributes\n");
fprintf(stderr, " -z attributes String to insert as td or th attributes\n");
@@ -1931,7 +1964,7 @@
/* ------------------------------------------------------- */
-/* read data to POST from file, save contents and length */
+/* read data to POST/PUT from file, save contents and length */
static int open_postfile(const char *pfile)
{
@@ -1942,26 +1975,26 @@
rv = apr_file_open(&postfd, pfile, APR_READ, APR_OS_DEFAULT, cntxt);
if (rv != APR_SUCCESS) {
- fprintf(stderr, "ab: Could not open POST data file (%s): %s\n", pfile,
+ fprintf(stderr, "ab: Could not open data file (%s): %s\n", pfile,
apr_strerror(rv, errmsg, sizeof errmsg));
return rv;
}
rv = apr_file_info_get(&finfo, APR_FINFO_NORM, postfd);
if (rv != APR_SUCCESS) {
- fprintf(stderr, "ab: Could not stat POST data file (%s): %s\n", pfile,
+ fprintf(stderr, "ab: Could not stat data file (%s): %s\n", pfile,
apr_strerror(rv, errmsg, sizeof errmsg));
return rv;
}
postlen = (apr_size_t)finfo.size;
postdata = malloc(postlen);
if (!postdata) {
- fprintf(stderr, "ab: Could not allocate POST data buffer\n");
+ fprintf(stderr, "ab: Could not allocate data buffer\n");
return APR_ENOMEM;
}
rv = apr_file_read_full(postfd, postdata, postlen, NULL);
if (rv != APR_SUCCESS) {
- fprintf(stderr, "ab: Could not read POST data file: %s\n",
+ fprintf(stderr, "ab: Could not read data file: %s\n",
apr_strerror(rv, errmsg, sizeof errmsg));
return rv;
}
@@ -2016,7 +2049,7 @@
#endif
apr_getopt_init(&opt, cntxt, argc, argv);
- while ((status = apr_getopt(opt, "n:c:t:b:T:p:v:rkVhwix:y:z:C:H:P:A:g:X:de:Sq"
+ while ((status = apr_getopt(opt, "n:c:t:b:T:p:v:rkVhwix:y:z:C:H:P:A:g:X:de:SqDu:"
#ifdef USE_SSL
"Z:f:"
#endif
@@ -2041,9 +2074,12 @@
windowsize = atoi(optarg);
break;
case 'i':
- if (posting == 1)
- err("Cannot mix POST and HEAD\n");
- posting = -1;
+ if (method != 0) err("Cannot mix HTTP methods\n");
+ method = -1;
+ break;
+ case 'D':
+ if (method != 0) err("Cannot mix HTTP methods\n");
+ method = -2;
break;
case 'g':
gnuplot = strdup(optarg);
@@ -2058,10 +2094,20 @@
confidence = 0;
break;
case 'p':
- if (posting != 0)
- err("Cannot mix POST and HEAD\n");
+ if (method != 0)
+ err("Cannot mix HTTP methods\n");
+ if (0 == (r = open_postfile(optarg))) {
+ method = 1;
+ }
+ else if (postdata) {
+ exit(r);
+ }
+ break;
+ case 'u':
+ if (method != 0)
+ err("Cannot mix HTTP methods\n");
if (0 == (r = open_postfile(optarg))) {
- posting = 1;
+ method = 2;
}
else if (postdata) {
exit(r);
A => +275 -0
@@ 0,0 1,275 @@
+
+This patch is intended for ApacheBench version 2.3, which is distributed
+with Apache 2.2.9.
+
+
+--- ab.c.orig 2008-06-19 21:59:04.000000000 -0700
++++ ab.c 2008-06-19 22:44:25.000000000 -0700
+@@ -84,6 +84,10 @@
+ ** Version 2.3
+ ** SIGINT now triggers output_results().
+ ** Contributed by colm, March 30, 2006
++ **
++ ** Version 2.3.1
++ ** PUT and DELETE HTTP method support.
++ ** Contributed by Mahlon E. Smith <mahlon@martini.nu>, June 2008.
+ **/
+
+ /* Note: this version string should start with \d+[\d\.]* and be a valid
+@@ -95,7 +99,7 @@
+ * ab - or to due to a change in the distribution it is compiled with
+ * (such as an APR change in for example blocking).
+ */
+-#define AP_AB_BASEREVISION "2.3"
++#define AP_AB_BASEREVISION "2.3.1"
+
+ /*
+ * BUGS:
+@@ -261,7 +265,7 @@
+
+ int verbosity = 0; /* no verbosity by default */
+ int recverrok = 0; /* ok to proceed after socket receive errors */
+-int posting = 0; /* GET by default */
++int method = 0; /* -2 => DELETE, -1 => HEAD, 0 => GET (default), 1 => POST, 2 => PUT */
+ int requests = 1; /* Number of requests to make */
+ int heartbeatres = 100; /* How often do we say we're alive */
+ int concurrency = 1; /* Number of multiple requests to make */
+@@ -631,7 +635,7 @@
+ c->connect = tnow;
+ c->rwrote = 0;
+ c->rwrite = reqlen;
+- if (posting)
++ if (method)
+ c->rwrite += postlen;
+ }
+ else if (tnow > c->connect + aprtimeout) {
+@@ -757,8 +761,10 @@
+ if (keepalive)
+ printf("Keep-Alive requests: %d\n", doneka);
+ printf("Total transferred: %" APR_INT64_T_FMT " bytes\n", totalread);
+- if (posting > 0)
++ if (method == 1)
+ printf("Total POSTed: %" APR_INT64_T_FMT "\n", totalposted);
++ if (method == 2)
++ printf("Total PUT: %" APR_INT64_T_FMT "\n", totalposted);
+ printf("HTML transferred: %" APR_INT64_T_FMT " bytes\n", totalbread);
+
+ /* avoid divide by zero */
+@@ -771,7 +777,7 @@
+ (double) timetaken * 1000 / done);
+ printf("Transfer rate: %.2f [Kbytes/sec] received\n",
+ (double) totalread / 1024 / timetaken);
+- if (posting > 0) {
++ if (method > 0) {
+ printf(" %.2f kb/s sent\n",
+ (double) totalposted / timetaken / 1024);
+ printf(" %.2f kb/s total\n",
+@@ -1042,10 +1048,14 @@
+ printf("<tr %s><th colspan=2 %s>Total transferred:</th>"
+ "<td colspan=2 %s>%" APR_INT64_T_FMT " bytes</td></tr>\n",
+ trstring, tdstring, tdstring, totalread);
+- if (posting > 0)
++ if (method == 1)
+ printf("<tr %s><th colspan=2 %s>Total POSTed:</th>"
+ "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
+ trstring, tdstring, tdstring, totalposted);
++ if (method == 2)
++ printf("<tr %s><th colspan=2 %s>Total PUT:</th>"
++ "<td colspan=2 %s>%" APR_INT64_T_FMT "</td></tr>\n",
++ trstring, tdstring, tdstring, totalposted);
+ printf("<tr %s><th colspan=2 %s>HTML transferred:</th>"
+ "<td colspan=2 %s>%" APR_INT64_T_FMT " bytes</td></tr>\n",
+ trstring, tdstring, tdstring, totalbread);
+@@ -1058,7 +1068,7 @@
+ printf("<tr %s><th colspan=2 %s>Transfer rate:</th>"
+ "<td colspan=2 %s>%.2f kb/s received</td></tr>\n",
+ trstring, tdstring, tdstring, (double) totalread / timetaken);
+- if (posting > 0) {
++ if (method > 0) {
+ printf("<tr %s><td colspan=2 %s> </td>"
+ "<td colspan=2 %s>%.2f kb/s sent</td></tr>\n",
+ trstring, tdstring, tdstring,
+@@ -1466,8 +1476,8 @@
+ cl = strstr(c->cbuff, "Content-length:");
+ if (cl) {
+ c->keepalive = 1;
+- /* response to HEAD doesn't have entity body */
+- c->length = posting >= 0 ? atoi(cl + 16) : 0;
++ /* response to HEAD and DELETE doesn't have entity body */
++ c->length = method >= 0 ? atoi(cl + 16) : 0;
+ }
+ /* The response may not have a Content-Length header */
+ if (!cl) {
+@@ -1532,6 +1542,7 @@
+ int i;
+ apr_status_t status;
+ int snprintf_res = 0;
++ char *verb = "GET";
+ #ifdef NOT_ASCII
+ apr_size_t inbytes_left, outbytes_left;
+ #endif
+@@ -1588,24 +1599,44 @@
+ }
+
+ /* setup request */
+- if (posting <= 0) {
++ if (method <= 0) {
++ switch (method) {
++ case -2:
++ verb = "DELETE";
++ break;
++ case -1:
++ verb = "HEAD";
++ break;
++ case 0:
++ verb = "GET";
++ break;
++ }
+ snprintf_res = apr_snprintf(request, sizeof(_request),
+ "%s %s HTTP/1.0\r\n"
+ "%s" "%s" "%s"
+ "%s" "\r\n",
+- (posting == 0) ? "GET" : "HEAD",
++ verb,
+ (isproxy) ? fullurl : path,
+ keepalive ? "Connection: Keep-Alive\r\n" : "",
+ cookie, auth, hdrs);
+ }
+ else {
++ switch (method) {
++ case 1:
++ verb = "POST";
++ break;
++ case 2:
++ verb = "PUT";
++ break;
++ }
+ snprintf_res = apr_snprintf(request, sizeof(_request),
+- "POST %s HTTP/1.0\r\n"
++ "%s %s HTTP/1.0\r\n"
+ "%s" "%s" "%s"
+ "Content-length: %" APR_SIZE_T_FMT "\r\n"
+ "Content-type: %s\r\n"
+ "%s"
+ "\r\n",
++ verb,
+ (isproxy) ? fullurl : path,
+ keepalive ? "Connection: Keep-Alive\r\n" : "",
+ cookie, auth,
+@@ -1622,9 +1653,9 @@
+ reqlen = strlen(request);
+
+ /*
+- * Combine headers and (optional) post file into one contineous buffer
++ * Combine headers and (optional) post file into one continuous buffer
+ */
+- if (posting == 1) {
++ if (method >= 1) {
+ char *buff = malloc(postlen + reqlen + 1);
+ if (!buff) {
+ fprintf(stderr, "error creating request buffer: out of memory\n");
+@@ -1825,12 +1856,14 @@
+ fprintf(stderr, " -t timelimit Seconds to max. wait for responses\n");
+ fprintf(stderr, " -b windowsize Size of TCP send/receive buffer, in bytes\n");
+ fprintf(stderr, " -p postfile File containing data to POST. Remember also to set -T\n");
++ fprintf(stderr, " -u putfile File containing data to PUT.\n");
+ fprintf(stderr, " -T content-type Content-type header for POSTing, eg.\n");
+ fprintf(stderr, " 'application/x-www-form-urlencoded'\n");
+ fprintf(stderr, " Default is 'text/plain'\n");
+ fprintf(stderr, " -v verbosity How much troubleshooting info to print\n");
+ fprintf(stderr, " -w Print out results in HTML tables\n");
+- fprintf(stderr, " -i Use HEAD instead of GET\n");
++ fprintf(stderr, " -i Send a HEAD request\n");
++ fprintf(stderr, " -D Send a DELETE request\n");
+ fprintf(stderr, " -x attributes String to insert as table attributes\n");
+ fprintf(stderr, " -y attributes String to insert as tr attributes\n");
+ fprintf(stderr, " -z attributes String to insert as td or th attributes\n");
+@@ -1931,7 +1964,7 @@
+
+ /* ------------------------------------------------------- */
+
+-/* read data to POST from file, save contents and length */
++/* read data to POST/PUT from file, save contents and length */
+
+ static int open_postfile(const char *pfile)
+ {
+@@ -1942,26 +1975,26 @@
+
+ rv = apr_file_open(&postfd, pfile, APR_READ, APR_OS_DEFAULT, cntxt);
+ if (rv != APR_SUCCESS) {
+- fprintf(stderr, "ab: Could not open POST data file (%s): %s\n", pfile,
++ fprintf(stderr, "ab: Could not open data file (%s): %s\n", pfile,
+ apr_strerror(rv, errmsg, sizeof errmsg));
+ return rv;
+ }
+
+ rv = apr_file_info_get(&finfo, APR_FINFO_NORM, postfd);
+ if (rv != APR_SUCCESS) {
+- fprintf(stderr, "ab: Could not stat POST data file (%s): %s\n", pfile,
++ fprintf(stderr, "ab: Could not stat data file (%s): %s\n", pfile,
+ apr_strerror(rv, errmsg, sizeof errmsg));
+ return rv;
+ }
+ postlen = (apr_size_t)finfo.size;
+ postdata = malloc(postlen);
+ if (!postdata) {
+- fprintf(stderr, "ab: Could not allocate POST data buffer\n");
++ fprintf(stderr, "ab: Could not allocate data buffer\n");
+ return APR_ENOMEM;
+ }
+ rv = apr_file_read_full(postfd, postdata, postlen, NULL);
+ if (rv != APR_SUCCESS) {
+- fprintf(stderr, "ab: Could not read POST data file: %s\n",
++ fprintf(stderr, "ab: Could not read data file: %s\n",
+ apr_strerror(rv, errmsg, sizeof errmsg));
+ return rv;
+ }
+@@ -2016,7 +2049,7 @@
+ #endif
+
+ apr_getopt_init(&opt, cntxt, argc, argv);
+- while ((status = apr_getopt(opt, "n:c:t:b:T:p:v:rkVhwix:y:z:C:H:P:A:g:X:de:Sq"
++ while ((status = apr_getopt(opt, "n:c:t:b:T:p:v:rkVhwix:y:z:C:H:P:A:g:X:de:SqDu:"
+ #ifdef USE_SSL
+ "Z:f:"
+ #endif
+@@ -2041,9 +2074,12 @@
+ windowsize = atoi(optarg);
+ break;
+ case 'i':
+- if (posting == 1)
+- err("Cannot mix POST and HEAD\n");
+- posting = -1;
++ if (method != 0) err("Cannot mix HTTP methods\n");
++ method = -1;
++ break;
++ case 'D':
++ if (method != 0) err("Cannot mix HTTP methods\n");
++ method = -2;
+ break;
+ case 'g':
+ gnuplot = strdup(optarg);
+@@ -2058,10 +2094,20 @@
+ confidence = 0;
+ break;
+ case 'p':
+- if (posting != 0)
+- err("Cannot mix POST and HEAD\n");
++ if (method != 0)
++ err("Cannot mix HTTP methods\n");
++ if (0 == (r = open_postfile(optarg))) {
++ method = 1;
++ }
++ else if (postdata) {
++ exit(r);
++ }
++ break;
++ case 'u':
++ if (method != 0)
++ err("Cannot mix HTTP methods\n");
+ if (0 == (r = open_postfile(optarg))) {
+- posting = 1;
++ method = 2;
+ }
+ else if (postdata) {
+ exit(r);
A => experiments/ab_patches/ab_millisecond.patch +17 -0
@@ 0,0 1,17 @@
+
+This patch is intended for ApacheBench version 2.3.1, which is distributed
+with Apache 2.2.10. For some reason, millisecond resolution was removed
+from the tsv output format. This puts it back.
+
+
+--- ab.c 2008-11-17 08:42:58.000000000 -0800
++++ ab.c-milliseconds 2008-11-17 08:42:47.000000000 -0800
+@@ -983,7 +983,7 @@
+ fprintf(out, "%s\t%" APR_TIME_T_FMT "\t%" APR_TIME_T_FMT
+ "\t%" APR_TIME_T_FMT "\t%" APR_TIME_T_FMT
+ "\t%" APR_TIME_T_FMT "\n", tmstring,
+- apr_time_sec(stats[i].starttime),
++ stats[i].starttime,
+ ap_round_ms(stats[i].ctime),
+ ap_round_ms(stats[i].time - stats[i].ctime),
+ ap_round_ms(stats[i].time),
M lib/thingfish/config.rb +15 -14
@@ 24,7 24,6 @@
#---
#
# Please see the file LICENSE in the top-level directory for licensing details.
-
#
require 'tmpdir'
@@ 59,20 58,23 @@ class ThingFish::Config
# Define the layout and defaults for the underlying structs
DEFAULTS = {
- :ip => DEFAULT_BIND_IP,
- :port => DEFAULT_PORT,
- :user => nil,
- :datadir => DEFAULT_DATADIR,
- :spooldir => DEFAULT_SPOOLDIR,
- :bufsize => DEFAULT_BUFSIZE,
+ :ip => DEFAULT_BIND_IP,
+ :port => DEFAULT_PORT,
+ :user => nil,
+ :datadir => DEFAULT_DATADIR,
+ :spooldir => DEFAULT_SPOOLDIR,
+ :bufsize => DEFAULT_BUFSIZE,
+ :resource_dir => nil,
- :pipeline_max => 100,
+ :pipeline_max => 100,
:memory_bodysize_max => 100.kilobytes,
-
+ :connection_timeout => 30,
+
:profiling => {
- :enabled => false,
- :profile_dir => DEFAULT_PROFILEDIR,
- :metrics => [],
+ :enabled => false,
+ :connection_enabled => false,
+ :profile_dir => DEFAULT_PROFILEDIR,
+ :metrics => []
},
:use_strict_html_mimetype => false,
@@ 80,8 82,7 @@ class ThingFish::Config
:daemon => false,
:pidfile => nil,
:defaulthandler => {
- :html_index => 'index.rhtml',
- :resource_dir => nil,
+ :html_index => 'index.rhtml'
},
:plugins => {
M lib/thingfish/connectionmanager.rb +46 -28
@@ 23,6 23,8 @@
#
+require 'timeout'
+
require 'thingfish'
require 'thingfish/mixins'
require 'thingfish/request'
@@ 44,6 46,17 @@ class ThingFish::ConnectionManager
# The subdirectory to search for error templates
ERROR_TEMPLATE_DIR = Pathname.new( 'errors' )
+
+ # The default number of pipelined requests to handle before closing the
+ # connection.
+ DEFAULT_PIPELINE_MAX = 100
+
+ # The default number of seconds to wait for IO before closing the connection.
+ DEFAULT_TIMEOUT = 30
+
+ # The maximum number of bytes an entity body can be before it's spooled to disk
+ # instead of kept in memory
+ DEFAULT_MEMORY_BODYSIZE_MAX = 100.kilobytes
### Create a new ThingFish::ConnectionManager
@@ 52,16 65,13 @@ class ThingFish::ConnectionManager
@config = config
@dispatch_callback = dispatch_callback
@request_count = 0
-
- # Load resources for error templates from the defaulthandler's
- # currently configured resource directory.
- # TODO: top-level config for defaulthandler and I to share
- @resource_dir = @config.defaulthandler.resource_dir
-
+
+ @resource_dir = @config.resource_dir
@pipeline_max = @config.pipeline_max || DEFAULT_PIPELINE_MAX
@bufsize = @config.bufsize || DEFAULT_BUFSIZE
@spooldir = @config.spooldir_path
- @memory_bodysize_max = @config.memory_bodysize_max
+ @memory_bodysize_max = @config.memory_bodysize_max || DEFAULT_MEMORY_BODYSIZE_MAX
+ @timeout = @config.connection_timeout || DEFAULT_TIMEOUT
@read_buffer = ''
end
@@ 95,8 105,8 @@ class ThingFish::ConnectionManager
self.session_info
]
end
-
-
+
+
### Return human-readable information about the current session.
def session_info
return "%s (%d/%d requests)" % [
@@ 105,8 115,8 @@ class ThingFish::ConnectionManager
self.pipeline_max,
]
end
-
-
+
+
### Return human-readable information about the connection's socket
def peer_info
return "%s:%d" % [
@@ 114,13 124,11 @@ class ThingFish::ConnectionManager
self.socket.peeraddr[1],
]
end
-
### Read data from the socket when it becomes readable.
### :TODO: Chunked transfer-encoding
- ### :TODO: Force the connection closed after pipeline_max requests
- ### :TODO: Force the connection closed after a timeout
+ ### :TODO: Try nonblocking sysread/write instead of using the Timeout library
def process
done = false
@read_buffer = ''
@@ 133,37 141,43 @@ class ThingFish::ConnectionManager
response = self.dispatch_request( request )
self.write_response( response, request )
- done = !(request.keepalive? && response.keepalive?)
+ done = @request_count >= @config.pipeline_max ||
+ !(request.keepalive? && response.keepalive?)
self.log.debug "%s: Connection will %sstay alive" %
[ self.session_info, done ? "not " : "" ]
end
self.log.debug "%s: close" % [ self.session_info ]
- @socket.close
rescue Errno, IOError => err
self.log.error "%s: Error in connection" % [ self.session_info ]
+ # TODO: handle ECONNRESET, EINTR, EPIPE
+ rescue Timeout::Error => err
+ self.log.error "%s: timeout %s" % [ self.session_info, err.message ]
+ ensure
@socket.close
- # TODO: handle ECONNRESET, EINTR, EPIPE
end
### Read a ThingFish::Request from the socket and return it.
- ### :TODO: Time out if we can't read the request in a configurable number of seconds.
def read_request
until pos = @read_buffer.index( BLANK_LINE )
- @read_buffer << @socket.sysread( @bufsize )
- self.log.debug "Read buffer now has %d bytes." % [ @read_buffer.length ]
+ Timeout.timeout( @timeout ) do
+ @read_buffer << @socket.sysread( @bufsize )
+ self.log.debug "Read buffer now has %d bytes." % [ @read_buffer.length ]
+ end
end
headerdata = @read_buffer.slice!( 0, pos + BLANK_LINE.length )
request = ThingFish::Request.parse( headerdata, @config, @socket )
- # Add a body to the request if it's got a entity-body and it's not
+ # Add a body to the request if it's got a entity-body and it's not
# a shallow HTTP method
request.body = self.read_request_body( request )
return request
+ rescue Timeout::Error => err
+ raise err.exception( "while reading" )
end
@@ 193,7 207,9 @@ class ThingFish::ConnectionManager
cur_pos = body.tell
needed = body_length - cur_pos
- @read_buffer << @socket.sysread( @bufsize ) unless @read_buffer.length >= needed
+ Timeout.timeout( @timeout ) do
+ @read_buffer << @socket.sysread( @bufsize ) unless @read_buffer.length >= needed
+ end
until @read_buffer.empty? || needed <= 0
bytes = body.write( @read_buffer[0, needed] )
@@ 207,17 223,17 @@ class ThingFish::ConnectionManager
end
- ### Write the +response+ (a ThingFish::Response) for the given +request+ (a ThingFish::Request)
+ ### Write the +response+ (a ThingFish::Response) for the given +request+ (a ThingFish::Request)
### to the socket.
def write_response( response, request )
buffer = response.status_line + EOL +
- response.header_data + EOL
+ response.header_data + EOL
self.log.debug "Response buffer is: %p" % [ buffer ]
# Make the response body an IOish object if it isn't already
# and the request actually wants to see a body.
body = if request.http_method == :HEAD
- nil
+ nil
elsif response.body.respond_to?( :eof? )
response.body
else
@@ 226,11 242,13 @@ class ThingFish::ConnectionManager
# Write stuff in the buffer to the socket
until buffer.empty? && (body.nil? || body.eof?)
- buffer << body.read( @bufsize ) if
+ buffer << body.read( @bufsize ) if
buffer.length < @bufsize && !( body.nil? || body.eof? )
until buffer.empty?
- bytes = @socket.syswrite( buffer )
+ bytes = Timeout.timeout( @config.connection_timeout ) do
+ @socket.syswrite( buffer )
+ end
buffer.slice!( 0, bytes )
end
end
@@ 253,7 271,7 @@ class ThingFish::ConnectionManager
end
self.log.debug "Got %s response" % [ response.status_line ]
-
+
# Check to be sure we're providing a response which is acceptable to the client
unless response.body.nil? || ! response.status_is_successful?
unless response.content_type
M lib/thingfish/daemon.rb +100 -32
@@ 31,7 31,6 @@
#---
#
# Please see the file LICENSE in the top-level directory for licensing details.
-
#
begin
@@ 75,10 74,14 @@ class ThingFish::Daemon
# The SVN revision number
SVNRev = %q$Rev$
- # Options for the default handler
- DEFAULT_HANDLER_OPTIONS = {}
+
+ ### An exception class for signalling daemon threads that they need to shut down
+ class Shutdown < Exception; end
+ #################################################################
+ ### I N S T A N C E M E T H O D S
+ #################################################################
### Create a new ThingFish daemon object that will listen on the specified +ip+
### and +port+.
@@ 88,9 91,14 @@ class ThingFish::Daemon
@running = false
# Load the profiler library if profiling is enabled
- @have_profiling = false
+ @have_profiling = false
+ @profile_connections = false
if @config.profiling.enabled?
@have_profiling = self.load_profiler
+ if @config.profiling.connection_enabled?
+ @profile_connections = true
+ self.log.info "Enabled connection profiling for connections from localhost"
+ end
end
# Load plugins for filestore, filters, and handlers
@@ 98,6 106,7 @@ class ThingFish::Daemon
@filestore = @config.create_configured_filestore
@metastore = @config.create_configured_metastore
@filters = @config.create_configured_filters
+
@connection_threads = ThreadGroup.new
@supervisor = nil
@@ 112,6 121,7 @@ class ThingFish::Daemon
@listener = TCPServer.new( @config.ip, @config.port )
@listener.setsockopt( Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1 )
+ @listener_thread = nil
self.log.debug "Handler map is: %p" % [ @urimap ]
end
@@ 169,26 179,57 @@ class ThingFish::Daemon
Signal.trap( "INT" ) { self.shutdown("Interrupted") }
Signal.trap( "TERM" ) { self.shutdown("Terminated") }
+ @running = true
self.start_supervisor_thread
+ self.start_listener.join
+ end
+
- self.log.debug "Entering listen loop"
- while @running
+ ### Start the thread responsible for handling incoming connections. Returns the new Thread
+ ### object.
+ def start_listener
+ @listener_thread = Thread.new do
+ Thread.current.abort_on_exception = true
+ self.log.debug "Entering listen loop"
+
begin
- socket = @listener.accept
- self.handle_connection( socket )
- rescue IOError => err
+ while @running
+ begin
+ # use nonblocking accept so closing the socket causes the block to finish
+ socket = @listener.accept_nonblock
+ self.handle_connection( socket )
+ rescue Errno::EAGAIN, Errno::ECONNABORTED, Errno::EPROTO, Errno::EINTR
+ self.log.debug "No connections pending; waiting for listener to become readable."
+ IO.select([ @listener ])
+ next
+ rescue IOError => err
+ @running = false
+ self.log.info "Listener got a %s: exiting accept loop" % [ err.class.name ]
+ end
+ end
+ rescue Shutdown
+ self.log.info "Got the shutdown signal."
+ ensure
@running = false
- self.log.info "Listener got a %s: exiting accept loop" % [ err.class.name ]
+ @listener.close
end
+
+ self.log.debug "Exited listen loop"
end
- self.log.debug "Exited listen loop"
+ self.log.debug "Listener thread is: %p" % [ @listener_thread ]
+ return @listener_thread
+ rescue Spec::Mocks::MockExpectationError
+ # Re-raise expectation failures while testing
+ raise
+ rescue Exception => err
+ self.log.error "Unhandled %s: %s" % [ err.class.name, err.message ]
+ self.log.debug " %s" % [ err.backtrace.join("\n ") ]
end
### Start a thread that monitors connection threads and cleans them up when they're done.
def start_supervisor_thread
- @running = true
self.log.info "Starting supervisor thread"
# TODO: work crew thread collection
@@ 197,7 238,7 @@ class ThingFish::Daemon
while @running || ! @connection_threads.list.empty?
@connection_threads.list.find_all {|t| ! t.alive? }.each do |t|
- self.log.debug "Joining %p" % [ t ]
+ self.log.debug "Finishing up connection thread %p" % [ t ]
t.join
end
sleep 0.5
@@ 206,18 247,30 @@ class ThingFish::Daemon
self.log.info "Supervisor thread exiting"
end
- self.log.info "Supervisor thread is running (%p)" % [ @supervisor ]
+ self.log.debug "Supervisor thread is running (%p)" % [ @supervisor ]
end
### Handle an incoming connection on the given +socket+.
def handle_connection( socket )
connection = ThingFish::ConnectionManager.new( socket, @config, &self.method(:dispatch) )
- self.log.info "Connect: %s" % [ connection.session_info ]
+ self.log.info "Connect: %s (%p)" %
+ [ connection.session_info, Thread.current ]
t = Thread.new do
Thread.current.abort_on_exception = true
- connection.process
+ begin
+ if @profile_connections && socket.peeraddr[3] == '127.0.0.1'
+ self.profile( "connection" ) do
+ connection.process
+ end
+ else
+ connection.process
+ end
+ rescue Shutdown
+ self.log.info "Forceful shutdown of connection thread %p." % [ Thread.current ]
+ end
+ self.log.debug "Connection thread %p done." % [ Thread.current ]
end
@connection_threads.add( t )
end
@@ 226,26 279,30 @@ class ThingFish::Daemon
### Shut the server down gracefully, outputting the specified +reason+ for the
### shutdown to the logs.
def shutdown( reason="no reason", force=false )
- @running = false
- @listener.close
+ if @running
+ @running = false
+ else
+ force = true
+ end
+
+ self.log.info "Shutting down %p from %p" % [ @listener_thread, Thread.current ]
+ @listener_thread.raise( Shutdown ) if ! @listener_thread.nil? && @listener_thread.alive?
if force
- self.log.warn "Forceful shutdown. Terminating %d active requests." %
+ self.log.warn "Forceful shutdown. Terminating %d active connection threads." %
[ @connection_threads.list.length ]
+ self.log.debudebug " Active threads: %p" % [ @connection_threads.list ]
@connection_threads.list.each do |t|
t.raise( Shutdown )
end
else
- self.log.warn "Graceful shutdown. Waiting on %d active requests." %
+ self.log.warn "Graceful shutdown. Waiting on %d active connection threads." %
[ @connection_threads.list.length ]
+ self.log.debug " Active threads: %p" % [ @connection_threads.list ]
end
- until @connection_threads.list.empty?
- @connection_threads.list.select {|t| ! t.alive? }.each do |t|
- t.join
- end
- end
+ @supervisor.join
end
@@ 380,7 437,7 @@ class ThingFish::Daemon
def dispatch( request )
response = ThingFish::Response.new( request.http_version, @config )
- self.profile( request, response ) do
+ self.profile_request( request, response ) do
# Let the filters manhandle the request
self.filter_request( request, response )
@@ 396,11 453,22 @@ class ThingFish::Daemon
end
+ ### Start a profile for the provided +block+ if profiling is enabled for the
+ ### specified +request+.
+ def profile_request( request, response, &block )
+ if self.have_profiling and request.run_profile?
+ description = request.uri.path.gsub( %r{/}, '_' ).sub( /^_/, '' ).chomp('_')
+ self.profile( description, &block )
+ else
+ yield
+ end
+ end
+
+
### Start a profile for the provided +block+ if profiling is enabled, dumping the result
### out to a profile directory afterwards.
- def profile( request, response, &block )
-
- if self.have_profiling and request.run_profile?
+ def profile( description, &block )
+ if self.have_profiling
result = RubyProf.profile( &block )
self.log.debug {
@@ 413,7 481,7 @@ class ThingFish::Daemon
filename = @config.profiledir_path + "%f#%d#%s" % [
Time.now,
Thread.current.object_id * 2,
- request.uri.path.gsub( %r{/}, '_' ).sub( /^_/, '' ).chomp('_')
+ description
]
File.open( "#{filename}.html", 'w' ) do | profile |
printer = RubyProf::GraphHtmlPrinter.new( result )
@@ 440,7 508,7 @@ class ThingFish::Daemon
@config.profiling.metrics.include?( 'memory' ) and ! RubyProf::MEMORY.nil?
RubyProf.measure_mode = mask
- self.log.debug "Enabled profiling"
+ self.log.info "Enabled profiling"
return true
rescue Exception => err
@@ 536,7 604,7 @@ class ThingFish::Daemon
### Create an instance of the default handler, with options dictated by the
### specified +config+ object.
def create_default_handler( config )
- options = DEFAULT_HANDLER_OPTIONS.dup
+ options = { :resource_dir => config.resource_dir }
if config.defaulthandler
options.merge!( config.defaulthandler )
end
M lib/thingfish/handler.rb +1 -2
@@ 25,7 25,6 @@
#---
#
# Please see the file LICENSE in the top-level directory for licensing details.
-
#
begin
@@ 88,7 87,7 @@ class ThingFish::Handler
@daemon = nil
@options = options
@path = path
-
+
super()
self.log.debug "%s created with resource dir at: %p" % [ self.class.name, @resource_dir ]
M lib/thingfish/handler/default.rb +2 -2
@@ 52,9 52,9 @@ class ThingFish::DefaultHandler < ThingF
# Config defaults
CONFIG_DEFAULTS = {
- :html_index => 'index.rhtml',
+ :html_index => 'index.rhtml',
:cache_expiration => 30.minutes,
- :resource_dir => nil, # Use the ThingFish::Handler default
+ :resource_dir => nil, # Use the ThingFish::Handler default
}
# Pattern to match UUIDs more efficiently than uuidtools
M lib/thingfish/handler/staticcontent.rb +4 -4
@@ 71,20 71,20 @@ class ThingFish::StaticContentHandler <
def handle_get_request( path_info, request, response )
self.find_static_file( path_info, request, response )
end
-
-
+
+
### Check for static content that matches the +request+ if it isn't yet handled, and
### build a response if great success.
def delegate( request, response )
yield( request, response )
-
+
unless response.is_handled?
path_info = self.path_info( request )
self.log.debug "Okay, static handling for %s" % [ path_info ]
self.find_static_file( path_info, request, response )
end
end
-
+
### Safely fetch a request for +path_info+ that is a file on disk.
### Return a IO object to the file after sanity checking and adding cache headers.
M misc/rake/benchmark.rb +4 -3
@@ 36,9 36,10 @@ begin
config.logging.level = 'error'
config.logging.logfile = 'stderr'
config.plugins.filestore.maxsize = TESTIMAGE.size * 1000
- config.plugins.handlers <<
- {"simplemetadata"=>{"resource_dir"=>"var/www", "uris"=>"/metadata"}} <<
- {"simplesearch"=>{"resource_dir"=>"var/www", "uris"=>"/search"}}
+ config.plugins.urimap = {
+ '/metadata' => [{ 'simplemetadata' => {'resource_dir' => 'var/www'} }],
+ '/search' => [{ 'simplesearch' => {'resource_dir' => 'var/www'} }]
+ }
config.plugins.filters << ['ruby', 'yaml']
end
M misc/rake/lib/benchmarktask.rb +19 -7
@@ 48,6 48,7 @@ module Benchmark
@options = options
@times = []
+ @sorted_times = nil
end
@@ 74,8 75,19 @@ module Benchmark
attr_reader :options
+ ### Return the timed request data sorted cronologically.
+ def times
+ if @sorted_times.nil?
+ @sorted_times = @times.sort_by {|row| row[EPOCHTIME] }
+ end
+
+ return @sorted_times
+ end
+
+
### Append a row of output from 'ab' to the times for this datapoint
def <<( row )
+ @sorted_times = nil
@times << row.chomp.split( /\t/ )[ 1..-1 ].collect {|i| Integer(i) }
return self
end
@@ 83,14 95,14 @@ module Benchmark
### Return the Time of the start of the benchmark
def start_time
- epochseconds = @times.first[EPOCHTIME] / 1_000_000.0
+ epochseconds = self.times.first[EPOCHTIME] / 1_000_000.0
return Time.at( epochseconds )
end
### Return the Time of the end of the benchmark
def finish_time
- epochseconds = @times.last[EPOCHTIME] / 1_000_000.0
+ epochseconds = self.times.last[EPOCHTIME] / 1_000_000.0
return Time.at( epochseconds )
end
@@ 125,22 137,22 @@ module Benchmark
raise ScriptError, "no column constant called #{column.to_s.upcase}"
define_method( "#{name}_times" ) do
- @times.transpose[ columnidx ]
+ self.times.transpose[ columnidx ]
end
alias_method "#{column}s", "#{name}_times" unless name == column
define_method( "min_#{name}_time" ) do
- @times.transpose[columnidx].min
+ self.times.transpose[columnidx].min
end
alias_method "min_#{column}", "min_#{name}_time"
define_method( "max_#{name}_time" ) do
- @times.transpose[columnidx].max
+ self.times.transpose[columnidx].max
end
alias_method "max_#{column}", "max_#{name}_time"
define_method( "mean_#{name}_time" ) do
- @times.transpose[columnidx].inject(0) {|sum,n| sum + n } / self.count.to_f
+ self.times.transpose[columnidx].inject(0) {|sum,n| sum + n } / self.count.to_f
end
alias_method "mean_#{column}", "mean_#{name}_time"
@@ 150,7 162,7 @@ module Benchmark
alias_method "#{column}_standard_deviation", "#{name}_time_standard_deviation"
define_method( "#{name}_time_histogram" ) do
- return @times.transpose[columnidx].inject({}) {|hist,n|
+ return self.times.transpose[columnidx].inject({}) {|hist,n|
hist[ n ] ||= 0
hist[ n ] += 1
hist
M spec/thingfish/connectionmanager_spec.rb +47 -0
@@ 91,6 91,8 @@ describe ThingFish::ConnectionManager do
halfway = TESTING_GET_REQUEST.length / 2
chunker1, chunker2 = TESTING_GET_REQUEST[0..halfway], TESTING_GET_REQUEST[halfway + 1..-1]
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).twice.
+ and_yield
@socket.should_receive( :sysread ).with( @config.bufsize ).
and_return( chunker1, chunker2 )
ThingFish::Request.should_receive( :parse ).with( TESTING_GET_REQUEST, @config, @socket ).
@@ 104,6 106,8 @@ describe ThingFish::ConnectionManager do
it "returns a BAD_REQUEST response if a GET request has a body" do
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).
+ and_yield
@socket.should_receive( :sysread ).with( @config.bufsize ).
and_return( TESTING_GET_REQUEST )
ThingFish::Request.should_receive( :parse ).with( TESTING_GET_REQUEST, @config, @socket ).
@@ 126,6 130,9 @@ describe ThingFish::ConnectionManager do
first_half = TEST_CONTENT[ 0..len/2 ]
second_half = TEST_CONTENT[ (len/2 + 1)..-1 ]
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).twice.
+ and_yield
+
@request.should_receive( :header ).at_least(:once).
and_return({:content_length => len})
@request.should_receive( :http_method ).and_return( :POST )
@@ 153,6 160,9 @@ describe ThingFish::ConnectionManager do
first_half = TEST_CONTENT[ 0..len/2 ]
second_half = TEST_CONTENT[ (len/2 + 1)..-1 ]
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).twice.
+ and_yield
+
@request.should_receive( :header ).at_least(:once).
and_return({:content_length => len})
@request.should_receive( :http_method ).and_return( :POST )
@@ 436,6 446,9 @@ describe ThingFish::ConnectionManager do
io.should_receive( :read ).with( an_instance_of(Fixnum) ).
and_return( first_half, second_half )
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).twice.
+ and_yield
+
first_chunker = status_line + EOL +
header_data + EOL +
first_half
@@ 459,6 472,9 @@ describe ThingFish::ConnectionManager do
@response.should_not_receive( :body )
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).once.
+ and_yield
+
first_chunker = status_line + EOL +
header_data + EOL
@socket.should_receive( :syswrite ).with( first_chunker ).
@@ 490,6 506,9 @@ describe ThingFish::ConnectionManager do
io.should_receive( :read ).with( an_instance_of(Fixnum) ).
and_return( first_half, second_half )
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).twice.
+ and_yield
+
first_chunker = status_line + EOL +
header_data + EOL +
first_half
@@ 512,12 531,39 @@ describe ThingFish::ConnectionManager do
@response.should_receive( :keepalive? ).and_return( false )
@socket.should_receive( :close ).once
+ @socket.stub!( :peeraddr ).and_return( [] )
Timeout.timeout( 1 ) do
@conn.process
end
end
+ it "closes the connection if request count exceeds pipeline max" do
+ @config.pipeline_max = 2
+ @conn.instance_variable_set( :@request_count, 1 )
+
+ @conn.should_receive( :read_request ).once.and_return( @request )
+ @conn.should_receive( :dispatch_request ).with( @request ).once.and_return( @response )
+ @conn.should_receive( :write_response ).with( @response, @request ).once
+
+ @socket.should_receive( :close ).once
+ @socket.stub!( :peeraddr ).and_return( [] )
+
+ Timeout.timeout( 1 ) do
+ @conn.process
+ end
+ end
+
+ it "closes the connection on connection timeouts" do
+ @socket.should_receive( :close ).once
+ @socket.stub!( :peeraddr ).and_return( [] )
+
+ Timeout.should_receive( :timeout ).with( @config.connection_timeout ).
+ and_raise( TimeoutError.new("timeout") )
+
+ @conn.process
+ end
+
it "processes another request if both the request and response have pipelining enabled" do
@conn.should_receive( :read_request ).twice.and_return( @request )
@conn.should_receive( :dispatch_request ).with( @request ).twice.and_return( @response )
@@ 527,6 573,7 @@ describe ThingFish::ConnectionManager do
@response.should_receive( :keepalive? ).and_return( true, false )
@socket.should_receive( :close ).once
+ @socket.stub!( :peeraddr ).and_return( [] )
Timeout.timeout( 1 ) do
@conn.process
M spec/thingfish/daemon_spec.rb +92 -39
@@ 1,4 1,3 @@
-
#!/usr/bin/env ruby
BEGIN {
@@ 50,9 49,6 @@ describe ThingFish::Daemon do
# listening on a real socket.
@listener = stub( "Server listener socket", :setsockopt => nil )
TCPServer.stub!( :new ).and_return( @listener )
-
- @connection_thread = mock( "connection thread" )
- Thread.stub!( :new ).and_return( @connection_thread )
end
# Daemon tests the logger interface, so we reset after each test
@@ 110,41 106,6 @@ describe ThingFish::Daemon do
end
- ### Startup
-
- it "starts accepting connections after starting its filestore and metastore when started" do
- config = ThingFish::Config.new
- daemon = ThingFish::Daemon.new( config )
-
- Process.should_receive( :euid ).at_least( :once ).and_return( 266 )
-
- filestore = mock( "filestore", :null_object => true )
- daemon.instance_variable_set( :@filestore, filestore )
- metastore = mock( "metastore", :null_object => true )
- daemon.instance_variable_set( :@metastore, metastore )
-
- filestore.should_receive( :on_startup )
- metastore.should_receive( :on_startup )
-
- conn_threadgroup = mock( "connection threadgroup" )
- daemon.instance_variable_set( :@connection_threads, conn_threadgroup )
-
- socket = mock( "client socket" )
- @listener.should_receive( :accept ).and_return( socket )
- socket.should_receive( :peeraddr ).and_return([1,2,3])
- ThingFish::ConnectionManager.should_receive( :new ).with( socket, config )
- conn_threadgroup.should_receive( :add ).with( @connection_thread ).
- and_return { daemon.shutdown }
-
- # For the shutdown call
- @listener.should_receive( :close )
- conn_threadgroup.stub!( :list ).and_return( [] )
-
- daemon.start
- end
-
-
-
### End-to-end
describe " configured with defaults" do
@@ 185,9 146,95 @@ describe ThingFish::Daemon do
@response.stub!( :body ).and_return( "test body" )
@processor.stub!( :process ).and_return( @request, @response )
+
+ @supervisor_thread = mock( "supervisor thread" )
+ @connection_thread = mock( "connection thread" )
+ @listener_thread = mock( "listener thread" )
end
+ describe "startup" do
+
+ it "starts its filestore, metastore, and listener thread on startup" do
+ Process.should_receive( :euid ).at_least( :once ).and_return( 266 )
+
+ filestore = mock( "filestore", :null_object => true )
+ @daemon.instance_variable_set( :@filestore, filestore )
+ metastore = mock( "metastore", :null_object => true )
+ @daemon.instance_variable_set( :@metastore, metastore )
+
+ filestore.should_receive( :on_startup )
+ metastore.should_receive( :on_startup )
+
+ Thread.should_receive( :new ).and_return( @supervisor_thread, @listener_thread )
+ @listener_thread.should_receive( :join )
+
+ @daemon.start
+ end
+
+
+ it "the listener thread accepts and creates a ConnectionManager for each incoming connection" do
+ conn_threadgroup = mock( "connection threadgroup" )
+ @daemon.instance_variable_set( :@connection_threads, conn_threadgroup )
+ @daemon.instance_variable_set( :@running, true )
+ @daemon.instance_variable_set( :@supervisor, @supervisor_thread )
+ @supervisor_thread.stub!( :join )
+
+ Thread.should_receive( :new ).and_yield.
+ and_return( @connection_thread, @listener_thread )
+ Thread.stub!( :current ).and_return( @connection_thread, @listener_thread )
+ @listener_thread.should_receive( :abort_on_exception= ).with( true )
+ @connection_thread.should_receive( :abort_on_exception= ).with( true )
+
+ socket = mock( "client socket" )
+ @listener.should_receive( :accept_nonblock ).and_return( socket )
+
+ connection = mock( "connection manager" )
+ ThingFish::ConnectionManager.should_receive( :new ).
+ with( socket, @config ).
+ and_return( connection )
+
+ connection.stub!( :session_info )
+ connection.should_receive( :process )
+
+ conn_threadgroup.should_receive( :add ).with( @connection_thread ).
+ and_return { @daemon.shutdown }
+
+ # For the shutdown call
+ @listener.should_receive( :close )
+ conn_threadgroup.stub!( :list ).and_return( [] )
+
+ @daemon.start_listener
+ end
+
+
+ it "doesn't block if the listener socket doesn't have an incoming connection" do
+ conn_threadgroup = mock( "connection threadgroup" )
+ @daemon.instance_variable_set( :@connection_threads, conn_threadgroup )
+ @daemon.instance_variable_set( :@running, true )
+ @daemon.instance_variable_set( :@supervisor, @supervisor_thread )
+ @supervisor_thread.stub!( :join )
+
+ Thread.should_receive( :new ).and_yield.and_return( @listener_thread )
+ Thread.stub!( :current ).and_return( @connection_thread, @listener_thread )
+ @connection_thread.should_receive( :abort_on_exception= ).with( true )
+
+ socket = mock( "client socket" )
+ @listener.should_receive( :accept_nonblock ).and_raise( Errno::EAGAIN )
+
+ ThingFish::ConnectionManager.should_not_receive( :new )
+
+ IO.should_receive( :select ).with([ @listener ]).
+ and_return { @daemon.instance_variable_set(:@running, false) }
+
+ # For the shutdown call
+ @listener.should_receive( :close )
+ conn_threadgroup.stub!( :list ).and_return( [] )
+
+ @daemon.start_listener
+ end
+ end
+
### Handlers
describe " handler dispatching" do
@@ 621,10 668,16 @@ describe ThingFish::Daemon do
@config = ThingFish::Config.new
@config.user = 'not-root'
@daemon = ThingFish::Daemon.new( @config )
+
+ @listener_thread = mock( "listener thread" )
end
it "drops privileges" do
+ @daemon.should_receive( :start_supervisor_thread )
+ @daemon.should_receive( :start_listener ).and_return( @listener_thread )
+ @listener_thread.should_receive( :join )
+
Process.should_receive( :euid ).at_least( :once ).and_return(0)
pwent = mock( 'not-root pw entry' )