Commit 36033a0d authored by Gregory Potamianos's avatar Gregory Potamianos

Imported Upstream version 1.9

parent 3ddc3306
......@@ -2,5 +2,4 @@
vers.c
ct/_*
beanstalkd
beanstalkd-*.tar.gz
NEWS.md
/News
......@@ -2,11 +2,11 @@ PREFIX=/usr/local
BINDIR=$(PREFIX)/bin
CFLAGS=-Wall -Werror\
-Wformat=2\
-g\
LDFLAGS=
OS=$(shell uname|tr A-Z a-z)
INSTALL=install
TAR=tar
VERS=$(shell ./vers.sh)
TARG=beanstalkd
......@@ -41,7 +41,6 @@ HFILES=\
CLEANFILES=\
vers.c\
$(TARG)-*.tar.gz\
.PHONY: all
all: $(TARG)
......@@ -88,24 +87,6 @@ endif
vers.c:
./verc.sh >vers.c
.PHONY: dist
dist: $(TARG)-$(VERS).tar.gz
$(TARG)-$(VERS).tar:
git archive -o $@ --prefix=$(TARG)-$(VERS)/ v$(VERS)
mkdir -p $(TARG)-$(VERS)/mk
echo 'printf "$(VERS)"' >$(TARG)-$(VERS)/vers.sh
chmod +x $(TARG)-$(VERS)/vers.sh
$(TAR) --append -f $@ $(TARG)-$(VERS)/vers.sh
sed 's/@VERSION@/$(VERS)/' <pkg/beanstalkd.spec.in >$(TARG)-$(VERS)/beanstalkd.spec
$(TAR) --append -f $@ $(TARG)-$(VERS)/beanstalkd.spec
cp NEWS.md $(TARG)-$(VERS)/NEWS.md
$(TAR) --append -f $@ $(TARG)-$(VERS)/NEWS.md
rm -r $(TARG)-$(VERS)
$(TARG)-$(VERS).tar.gz: $(TARG)-$(VERS).tar
gzip -f $<
doc/beanstalkd.1 doc/beanstalkd.1.html: doc/beanstalkd.ronn
ronn $<
......
This is beanstalkd version 1.8.
As always, there will be no incompatible protocol changes until
version 2.0. A client written for version 1.8 will work
unmodified with any later 1.x release of beanstalkd.
News
----
- correctly log command names in verbose mode
- correctly check for hangup during reserve
- add protocol.md
- fix tests on freebsd
- stricter format checking; fix format call errors
- Tolerate compilation without a Git repository.
- New command: kick-job.
- document missing tube stats fields; fixes #127
Full list of changes (includes authorship information):
<http://github.com/kr/beanstalkd/compare/v1.7...v1.8>
Our Urls
--------
Download the 1.8 tarball directly:
<https://github.com/downloads/kr/beanstalkd/beanstalkd-1.8.tar.gz>
Learn all about beanstalk:
<http://kr.github.com/beanstalkd/>
Talk about beanstalk development or use at:
<http://groups.google.com/group/beanstalk-talk>
Bugs
----
Please report any bugs to:
<http://github.com/kr/beanstalkd/issues>
This is beanstalkd version 1.9.
As always, there will be no incompatible protocol changes until
version 2.0. A client written for version 1.9 will work
unmodified with any later 1.x release of beanstalkd.
News
----
- compile with debugging symbols by default
- fix case where a job would not time out properly (#174)
- fix build on hurd (#177)
- document the delayed->ready transition via 'kick' (#176)
- fix crash from Conn use after free and double-free (#159)
- fix high CPU use during heavy system load (#169)
- add a unique id and hostname to stats output (#156)
Full list of changes (includes authorship information):
<http://github.com/kr/beanstalkd/compare/v1.8...v1.9>
Our Urls
--------
Download the 1.9 tarball directly:
<https://github.com/kr/beanstalkd/archive/v1.9.tar.gz>
Learn all about beanstalk:
<http://kr.github.com/beanstalkd/>
Talk about beanstalk development or use at:
<http://groups.google.com/group/beanstalk-talk>
Bugs
----
Please report any bugs to:
<http://github.com/kr/beanstalkd/issues>
This is beanstalkd, a fast, general-purpose work queue.
See http://kr.github.com/beanstalkd/ for general info.
See http://kr.github.io/beanstalkd/ for general info.
QUICK START
......@@ -12,7 +12,7 @@ also try,
$ ./beanstalkd -h
$ ./beanstalkd -VVV
$ make CFLAGS=-g
$ make CFLAGS=-O2
$ make CC=clang
$ make check
$ make install
......@@ -39,7 +39,7 @@ Unit tests are in *-test.c. See https://github.com/kr/ct for
information on how to write them.
Copyright © 2007-2012 the authors of beanstalkd.
Copyright © 2007-2013 the authors of beanstalkd.
Copyright in contributions to beanstalkd is retained
by the original copyright holder of each contribution.
See file LICENSE for terms of use.
......@@ -4,7 +4,7 @@
%define beanstalkd_logdir %{_localstatedir}/log/beanstalkd
Name: beanstalkd
Version: 1.8
Version: 1.9
Release: 0%{?dist}
Summary: A simple, fast workqueue service
......
......@@ -241,6 +241,5 @@ connclose(Conn *c)
heapremove(&c->srv->conns, c->tickpos);
}
protrmdirty(c);
free(c);
}
// Requirements:
// #include <stdint.h>
// #include <stdlib.h>
typedef unsigned char uchar;
typedef uchar byte;
typedef unsigned int uint;
......@@ -243,8 +247,7 @@ extern size_t primes[];
extern size_t job_data_size_limit;
void prot_init(void);
void prottick(Server *s);
void protrmdirty(Conn*);
int64 prottick(Server *s);
Conn *remove_waiting_conn(Conn *c);
......
......@@ -61,7 +61,7 @@ Here is a picture with more possibilities:
put with delay release with delay
----------------> [DELAYED] <------------.
| |
| (time passes) |
kick | (time passes) |
| |
put v reserve | delete
-----------------> [READY] ---------> [RESERVED] --------> *poof*
......@@ -159,6 +159,10 @@ reserve-with-timeout <seconds>\r\n
This will return a newly-reserved job. If no job is available to be reserved, beanstalkd will wait to send a response until one becomes available. Once a job is reserved for the client, the client has limited time to run (TTR) the job before the job times out. When the job times out, the server will put the job back into the ready queue. Both the TTR and the actual time left can be found in response to the `stats-job` command.
If more than one job is ready, beanstalkd will choose the one with the
smallest priority value. Within each priority, it will choose the one that
was received first.
A timeout value of `0` will cause the server to immediately return either a response or `TIMED_OUT`. A positive value of timeout will limit the amount of time the client will block on the reserve request until a job becomes available.
##### `reserve` responses
......@@ -364,7 +368,7 @@ The response is one of:
* `NOT_FOUND\r\n` if the job does not exist or is not in a kickable state. This can also happen upon internal errors.
* `KICKED\r\n` when the operation succeeded.
#### `stats-jobs` command
#### `stats-job` command
The stats-job command gives statistical information about the specified job if it exists. Its form is:
......@@ -372,11 +376,11 @@ The stats-job command gives statistical information about the specified job if i
stats-job <id>\r\n
```
##### `stats-jobs` options
##### `stats-job` options
* `<id>` is a job id.
##### `stats-jobs` responses
##### `stats-job` responses
The response is one of:
......@@ -484,7 +488,7 @@ The stats data for the system is a YAML file representing a single dictionary of
* `cmd-list-tubes` is the cumulative number of list-tubes commands.
* `cmd-list-tube-used` is the cumulative number of list-tube-used commands.
* `cmd-list-tubes-watched` is the cumulative number of list-tubes-watched commands.
* `cmd-pause-tube` is the cumulative number of pause-tube commands
* `cmd-pause-tube` is the cumulative number of pause-tube commands.
* `job-timeouts` is the cumulative count of times a job has timed out.
* `total-jobs` is the cumulative count of jobs created.
* `max-job-size` is the maximum number of bytes in a job.
......@@ -499,11 +503,13 @@ The stats data for the system is a YAML file representing a single dictionary of
* `rusage-utime` is the cumulative user CPU time of this process in seconds and microseconds.
* `rusage-stime` is the cumulative system CPU time of this process in seconds and microseconds.
* `uptime` is the number of seconds since this server process started running.
* `binlog-oldest-index` is the index of the oldest binlog file needed to store the current jobs
* `binlog-current-index` is the index of the current binlog file being written to. If binlog is not active this value will be 0
* `binlog-max-size` is the maximum size in bytes a binlog file is allowed to get before a new binlog file is opened
* `binlog-records-written` is the cumulative number of records written to the binlog
* `binlog-records-migrated` is the cumulative number of records written as part of compaction
* `binlog-oldest-index` is the index of the oldest binlog file needed to store the current jobs.
* `binlog-current-index` is the index of the current binlog file being written to. If binlog is not active this value will be 0.
* `binlog-max-size` is the maximum size in bytes a binlog file is allowed to get before a new binlog file is opened.
* `binlog-records-written` is the cumulative number of records written to the binlog.
* `binlog-records-migrated` is the cumulative number of records written as part of compaction.
* `id` is a random id string for this server process, generated when each beanstalkd process starts.
* `hostname` is the hostname of the machine as determined by uname.
#### `list-tubes` command
......@@ -588,4 +594,4 @@ pause-tube <tube-name> <delay>\r\n
There are two possible responses:
* `PAUSED\r\n` to indicate success.
* `NOT_FOUND\r\n` if the tube does not exist.
\ No newline at end of file
* `NOT_FOUND\r\n` if the tube does not exist.
......@@ -206,6 +206,10 @@ job before the job times out. When the job times out, the server will put the
job back into the ready queue. Both the TTR and the actual time left can be
found in response to the stats-job command.
If more than one job is ready, beanstalkd will choose the one with the
smallest priority value. Within each priority, it will choose the one that
was received first.
A timeout value of 0 will cause the server to immediately return either a
response or TIMED_OUT. A positive value of timeout will limit the amount of
time the client will block on the reserve request until a job becomes
......@@ -587,7 +591,7 @@ beanstalkd process starts; they are not stored on disk with the -b flag.
- "cmd-list-tubes-watched" is the cumulative number of list-tubes-watched
commands.
- "cmd-pause-tube" is the cumulative number of pause-tube commands
- "cmd-pause-tube" is the cumulative number of pause-tube commands.
- "job-timeouts" is the cumulative count of times a job has timed out.
......@@ -623,19 +627,24 @@ beanstalkd process starts; they are not stored on disk with the -b flag.
- "uptime" is the number of seconds since this server process started running.
- "binlog-oldest-index" is the index of the oldest binlog file needed to
store the current jobs
store the current jobs.
- "binlog-current-index" is the index of the current binlog file being
written to. If binlog is not active this value will be 0
written to. If binlog is not active this value will be 0.
- "binlog-max-size" is the maximum size in bytes a binlog file is allowed
to get before a new binlog file is opened
to get before a new binlog file is opened.
- "binlog-records-written" is the cumulative number of records written
to the binlog
to the binlog.
- "binlog-records-migrated" is the cumulative number of records written
as part of compaction
as part of compaction.
- "id" is a random id string for this server process, generated when each
beanstalkd process starts.
- "hostname" the hostname of the machine as determined by uname.
The list-tubes command returns a list of all existing tubes. Its form is:
......
......@@ -187,7 +187,7 @@ readrec(File *f, job l, int *err)
if (!(j || namelen)) {
// We read a short record without having seen a
// full record for this job, so the full record
// was in an eariler file that has been deleted.
// was in an earlier file that has been deleted.
// Therefore the job itself has either been
// deleted or migrated; either way, this record
// should be ignored.
......
......@@ -340,6 +340,19 @@ cttestunderscore()
}
void
cttest2cmdpacket()
{
port = SERVER();
fd = mustdiallocal(port);
mustsend(fd, "use a\r\nuse b\r\n");
ckresp(fd, "USING a\r\n");
ckresp(fd, "USING b\r\n");
killsrv();
}
void
cttesttoobig()
{
......
......@@ -28,7 +28,6 @@ make_server_socket(char *host, char *port)
if (r > 1) {
twarnx("inherited more than one listen socket;"
" ignoring all but the first");
r = 1;
}
fd = SD_LISTEN_FDS_START;
r = sd_is_socket_inet(fd, 0, SOCK_STREAM, 1, 0);
......
---
layout: post
title: Beanstalkd @VERSION@ Release Notes
version: @VERSION@
dist: https://github.com/kr/beanstalkd/archive/v@VERSION@.tar.gz
file: beanstalkd-v@VERSION@.tar.gz
---
#!/usr/bin/env bash
set -e
set -o pipefail
exp() {
sed s/@VERSION@/$ver/ | sed s/@PARENT@/$prev/
}
clean() {
rm -f "$GIT_INDEX_FILE"
}
mkobj() {
git hash-object -w --stdin
}
die() {
echo >&2 "$@"
exit 2
}
ver=`./vers.sh`
case $ver in *+*) die bad ver $ver ;; esac
prev=`git describe --abbrev=0 --match=dev* --tags dev$ver^|sed s/^dev//`
test -n "$prev" || die no prev ver
test -f News || die no News
export GIT_INDEX_FILE
GIT_INDEX_FILE=`mktemp -t beanstalkd-dist-index`
trap clean EXIT
git read-tree dev$ver
newsobj=`cat News pkg/newstail.in|exp|mkobj`
versobj=`echo "printf '$ver'"|mkobj`
specobj=`exp <pkg/beanstalkd.spec.in|mkobj`
git update-index --add --cacheinfo 100644 $newsobj News
git update-index --cacheinfo 100755 $versobj vers.sh
git update-index --add --cacheinfo 100644 $specobj beanstalkd.spec
tree=`git write-tree`
commit=`git commit-tree $tree -p dev$ver -m "release $ver"`
git tag -m "beanstalkd version $ver" v$ver $commit
git rev-parse --verify gh-pages >/dev/null
parent=`git rev-parse --verify gh-pages`
git read-tree $parent
postobj=`(exp <pkg/bloghead.in; git cat-file blob v$ver:News)|mkobj`
post=_posts/`date +%Y-%m-%d`-$ver-release-notes.md
git update-index --add --cacheinfo 100644 $postobj $post
tree=`git write-tree`
commit=`git commit-tree $tree -p $parent -m "announce release $ver"`
git update-ref -m "commit: announce release $ver" refs/heads/gh-pages $commit $parent
#!/usr/bin/env bash
set -e
set -o pipefail
die() {
echo >&2 "$@"
exit 2
}
addr=beanstalk-talk@googlegroups.com
ver=`./vers.sh`
case $ver in *+*) die bad ver $ver ;; esac
(cat <<end; git cat-file -p v$ver:News)|msmtp -t $addr
To: $addr
From: `git config user.name` <`git config user.email`>
Subject: [ANN] beanstalkd $ver
end
Full list of changes (includes authorship information):
<http://github.com/kr/beanstalkd/compare/v@PARENT@...v@VERSION@>
Our Urls
--------
Download the @VERSION@ tarball directly:
<https://github.com/kr/beanstalkd/archive/v@VERSION@.tar.gz>
Learn all about beanstalk:
<http://kr.github.com/beanstalkd/>
Talk about beanstalk development or use at:
<http://groups.google.com/group/beanstalk-talk>
Bugs
----
Please report any bugs to:
<http://github.com/kr/beanstalkd/issues>
......@@ -8,6 +8,7 @@
#include <sys/resource.h>
#include <sys/uio.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <inttypes.h>
......@@ -108,6 +109,7 @@ size_t job_data_size_limit = JOB_DATA_SIZE_LIMIT_DEFAULT;
#define STATE_SENDWORD 3
#define STATE_WAIT 4
#define STATE_BITBUCKET 5
#define STATE_CLOSE 6
#define OP_UNKNOWN 0
#define OP_PUT 1
......@@ -183,6 +185,8 @@ size_t job_data_size_limit = JOB_DATA_SIZE_LIMIT_DEFAULT;
"binlog-records-migrated: %" PRId64 "\n" \
"binlog-records-written: %" PRId64 "\n" \
"binlog-max-size: %d\n" \
"id: %s\n" \
"hostname: %s\n" \
"\r\n"
#define STATS_TUBE_FMT "---\n" \
......@@ -231,6 +235,14 @@ static tube default_tube;
static int drain_mode = 0;
static int64 started_at;
enum {
NumIdBytes = 8
};
static char id[NumIdBytes * 2 + 1]; // hex-encoded len of NumIdBytes
static struct utsname node_info;
static uint64 op_ct[TOTAL_OPS], timeout_ct = 0;
static Conn *dirty;
......@@ -289,7 +301,7 @@ reply(Conn *c, char *line, int len, int state)
}
void
static void
protrmdirty(Conn *c)
{
Conn *x, *newdirty = NULL;
......@@ -372,6 +384,7 @@ reserve_job(Conn *c, job j)
j->r.state = Reserved;
job_insert(&c->reserved_jobs, j);
j->reserver = c;
c->pending_timeout = -1;
if (c->soonest_job && j->r.deadline_at < c->soonest_job->r.deadline_at) {
c->soonest_job = j;
}
......@@ -695,7 +708,7 @@ check_err(Conn *c, const char *s)
if (errno == EWOULDBLOCK) return;
twarn("%s", s);
connclose(c);
c->state = STATE_CLOSE;
return;
}
......@@ -917,7 +930,9 @@ fmt_stats(char *buf, size_t size, void *x)
wcur,
srv->wal.nmig,
srv->wal.nrec,
srv->wal.filesize);
srv->wal.filesize,
id,
node_info.nodename);
}
......@@ -1555,7 +1570,7 @@ dispatch_cmd(Conn *c)
reply_line(c, STATE_SENDWORD, "WATCHING %zu\r\n", c->watch.used);
break;
case OP_QUIT:
connclose(c);
c->state = STATE_CLOSE;
break;
case OP_PAUSE_TUBE:
op_ct[type]++;
......@@ -1671,7 +1686,10 @@ conn_data(Conn *c)
case STATE_WANTCOMMAND:
r = read(c->sock.fd, c->cmd + c->cmd_read, LINE_BUF_SIZE - c->cmd_read);
if (r == -1) return check_err(c, "read()");
if (r == 0) return connclose(c); /* the client hung up */
if (r == 0) {
c->state = STATE_CLOSE;
return;
}
c->cmd_read += r; /* we got some bytes */
......@@ -1696,7 +1714,10 @@ conn_data(Conn *c)
to_read = min(c->in_job_read, BUCKET_BUF_SIZE);
r = read(c->sock.fd, bucket, to_read);
if (r == -1) return check_err(c, "read()");
if (r == 0) return connclose(c); /* the client hung up */
if (r == 0) {
c->state = STATE_CLOSE;
return;
}
c->in_job_read -= r; /* we got some bytes */
......@@ -1711,7 +1732,10 @@ conn_data(Conn *c)
r = read(c->sock.fd, j->body + c->in_job_read, j->r.body_size -c->in_job_read);
if (r == -1) return check_err(c, "read()");
if (r == 0) return connclose(c); /* the client hung up */
if (r == 0) {
c->state = STATE_CLOSE;
return;
}
c->in_job_read += r; /* we got some bytes */
......@@ -1722,7 +1746,10 @@ conn_data(Conn *c)
case STATE_SENDWORD:
r= write(c->sock.fd, c->reply + c->reply_sent, c->reply_len - c->reply_sent);
if (r == -1) return check_err(c, "write()");
if (r == 0) return connclose(c); /* the client hung up */
if (r == 0) {
c->state = STATE_CLOSE;
return;
}
c->reply_sent += r; /* we got some bytes */
......@@ -1742,7 +1769,10 @@ conn_data(Conn *c)
r = writev(c->sock.fd, iov, 2);
if (r == -1) return check_err(c, "writev()");
if (r == 0) return connclose(c); /* the client hung up */
if (r == 0) {
c->state = STATE_CLOSE;
return;
}
/* update the sent values */
c->reply_sent += r;
......@@ -1810,6 +1840,10 @@ h_conn(const int fd, const short which, Conn *c)
conn_data(c);
while (cmd_data_ready(c) && (c->cmd_len = cmd_len(c))) do_cmd(c);
if (c->state == STATE_CLOSE) {
protrmdirty(c);
connclose(c);
}
update_conns();
}
......@@ -1819,7 +1853,7 @@ prothandle(Conn *c, int ev)
h_conn(c->sock.fd, ev, c);
}
void
int64
prottick(Server *s)
{
int r;
......@@ -1827,10 +1861,16 @@ prottick(Server *s)
int64 now;
int i;
tube t;
int64 period = 0x34630B8A000LL; /* 1 hour in nanoseconds */
int64 d;
now = nanoseconds();
while ((j = delay_q_peek())) {
if (j->r.deadline_at > now) break;
d = j->r.deadline_at - now;
if (d > 0) {
period = min(period, d);
break;
}
j = delay_q_take();
r = enqueue_job(s, j, 0, 0);
if (r < 1) bury_job(s, j, 0); /* out of memory, so bury it */
......@@ -1838,16 +1878,21 @@ prottick(Server *s)
for (i = 0; i < tubes.used; i++) {
t = tubes.items[i];
if (t->pause && t->deadline_at <= now) {
d = t->deadline_at - now;
if (t->pause && d <= 0) {
t->pause = 0;
process_queue();
}
else if (d > 0) {
period = min(period, d);
}
}
while (s->conns.len) {
Conn *c = s->conns.data[0];
if (c->tickat > now) {
d = c->tickat - now;
if (d > 0) {
period = min(period, d);
break;
}
......@@ -1856,6 +1901,8 @@ prottick(Server *s)
}
update_conns();
return period;
}
void
......@@ -1933,6 +1980,29 @@ prot_init()
started_at = nanoseconds();
memset(op_ct, 0, sizeof(op_ct));
int dev_random = open("/dev/urandom", O_RDONLY);
if (dev_random < 0) {
twarn("open /dev/urandom");
exit(50);
}
int i, r;
byte rand_data[NumIdBytes];
r = read(dev_random, &rand_data, NumIdBytes);
if (r != NumIdBytes) {
twarn("read /dev/urandom");
exit(50);
}
for (i = 0; i < NumIdBytes; i++) {
sprintf(id + (i * 2), "%02x", rand_data[i]);
}
close(dev_random);
if (uname(&node_info) == -1) {
warn("uname");
exit(50);
}
ms_init(&tubes, NULL, NULL);
TUBE_ASSIGN(default_tube, tube_find_or_make("default"));
......
......@@ -18,7 +18,7 @@ srvserve(Server *s)
{
int r;
Socket *sock;
int64 period = 10*1000000; // 10ms
int64 period;
if (sockinit() == -1) {
twarnx("sockinit");
......@@ -43,20 +43,15 @@ srvserve(Server *s)
}
int64 t = nanoseconds();
for (;;) {
period = prottick(s);
int rw = socknext(&sock, period);
if (rw == -1) {
twarnx("socknext");
exit(1);
}
int64 t1 = nanoseconds();
if (t1-t > period) {
prottick(s);
t = t1;
}
if (rw) {
sock->f(sock->x, rw);
}
......
printf "1.8"
printf '1.9'
......@@ -405,15 +405,18 @@ waldirlock(Wal *w)
int r;
int fd;
struct flock lk;
char path[PATH_MAX];
char *path;
size_t path_length;
r = snprintf(path, PATH_MAX, "%s/lock", w->dir);
if (r > PATH_MAX) {
twarnx("path too long: %s/lock", w->dir);
path_length = strlen(w->dir) + strlen("/lock") + 1;
if ((path = malloc(path_length)) == NULL) {
twarn("malloc");
return 0;
}
r = snprintf(path, path_length, "%s/lock", w->dir);
fd = open(path, O_WRONLY|O_CREAT, 0600);
free(path);
if (fd == -1) {
twarn("open");
return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment