diff --git a/Makefile b/Makefile index 3c768d7..d86558d 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ OBJS := $(TARGET:=.o) $(LIBOBJS) STATIC_LIB := libbuse.a CC := /usr/bin/gcc -CFLAGS := -g -pedantic -Wall -Wextra -std=c99 +override CFLAGS += -g -pedantic -Wall -Wextra -std=c99 LDFLAGS := -L. -lbuse -.PHONY: all clean +.PHONY: all clean test all: $(TARGET) $(TARGET): %: %.o $(STATIC_LIB) @@ -22,5 +22,9 @@ $(STATIC_LIB): $(LIBOBJS) $(LIBOBJS): %.o: %.c $(CC) $(CFLAGS) -o $@ -c $< +test: $(TARGET) + PATH=$(PWD):$$PATH sudo test/busexmp.sh + PATH=$(PWD):$$PATH sudo test/signal_termination.sh + clean: rm -f $(TARGET) $(OBJS) $(STATIC_LIB) diff --git a/README.md b/README.md index 520085d..ac44aa0 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ user. ## Running the Example Code -BUSE comes with an example driver in `busexmp.c` that implements a 128 MB +BUSE comes with an example driver in `busexmp.c` that implements a memory disk. To try out the example code, run `make` and then execute the following as root: modprobe nbd - ./busexmp /dev/nbd0 + ./busexmp 128M /dev/nbd0 You should then have an in-memory disk running, represented by the device file `/dev/nbd0`. You can create a file system on the virtual disk, mount it, and @@ -33,3 +33,27 @@ start reading and writing files on it: mkfs.ext4 /dev/nbd0 mount /dev/nbd0 /mnt + +BUSE should gracefuly disconnect from block device upon receiving SIGINT +or SIGTERM. However, if something goes wrong, block device is stuck in +unusable state and BUSE process exited or hung you can request +disconnect by: + + nbd-client -d /dev/nbd0 + +Actually this command performs clean disconnect and can also be used +to terminate running instance of BUSE. + +## Tests + +To perform checks you can run scripts in `test/` directory. They require: + * superuser previlages, + * nbd kernel module loaded, + * BUSE and nbd (`nbd-client`) binaries in PATH. + +`make test` will run all test scripts with BUSE added to PATH and using +sudo to grant permissions. + +To increase verbosity define `BUSE_DEBUG`. You can do this in make command: + + make test CFLAGS=-DBUSE_DEBUG diff --git a/buse.c b/buse.c index 4d52fe3..b8bb02a 100644 --- a/buse.c +++ b/buse.c @@ -17,21 +17,30 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ +#define _POSIX_C_SOURCE (200809L) + #include #include +#include #include -#include +#include #include +#include #include #include #include #include #include #include +#include #include #include "buse.h" +#ifndef BUSE_DEBUG + #define BUSE_DEBUG (0) +#endif + /* * These helper functions were taken from cliserv.h in the nbd distribution. */ @@ -80,10 +89,32 @@ static int write_all(int fd, char* buf, size_t count) return 0; } -int buse_main(const char* dev_file, const struct buse_operations *aop, void *userdata) -{ - int sp[2]; - int nbd, sk, err, tmp_fd; +/* Signal handler to gracefully disconnect from nbd kernel driver. */ +static int nbd_dev_to_disconnect = -1; +static void disconnect_nbd(int signal) { + (void)signal; + if (nbd_dev_to_disconnect != -1) { + if(ioctl(nbd_dev_to_disconnect, NBD_DISCONNECT) == -1) { + warn("failed to request disconect on nbd device"); + } else { + nbd_dev_to_disconnect = -1; + fprintf(stderr, "sucessfuly requested disconnect on nbd device\n"); + } + } +} + +/* Sets signal action like regular sigaction but is suspicious. */ +static int set_sigaction(int sig, const struct sigaction * act) { + struct sigaction oact; + int r = sigaction(sig, act, &oact); + if (r == 0 && oact.sa_handler != SIG_DFL) { + warnx("overriden non-default signal handler (%d: %s)", sig, strsignal(sig)); + } + return r; +} + +/* Serve userland side of nbd socket. If everything worked ok, return 0. */ +static int serve_nbd(int sk, const struct buse_operations * aop, void * userdata) { u_int64_t from; u_int32_t len; ssize_t bytes_read; @@ -91,71 +122,6 @@ int buse_main(const char* dev_file, const struct buse_operations *aop, void *use struct nbd_reply reply; void *chunk; - err = socketpair(AF_UNIX, SOCK_STREAM, 0, sp); - assert(!err); - - nbd = open(dev_file, O_RDWR); - if (nbd == -1) { - fprintf(stderr, - "Failed to open `%s': %s\n" - "Is kernel module `nbd' is loaded and you have permissions " - "to access the device?\n", dev_file, strerror(errno)); - return 1; - } - - if (aop->blksize) { - err = ioctl(nbd, NBD_SET_BLKSIZE, aop->blksize); - assert(err != -1); - } - if (aop->size) { - err = ioctl(nbd, NBD_SET_SIZE, aop->size); - assert(err != -1); - } - if (aop->size_blocks) { - err = ioctl(nbd, NBD_SET_SIZE_BLOCKS, aop->size_blocks); - assert(err != -1); - } - - err = ioctl(nbd, NBD_CLEAR_SOCK); - assert(err != -1); - - if (!fork()) { - /* The child needs to continue setting things up. */ - close(sp[0]); - sk = sp[1]; - - if(ioctl(nbd, NBD_SET_SOCK, sk) == -1){ - fprintf(stderr, "ioctl(nbd, NBD_SET_SOCK, sk) failed.[%s]\n", strerror(errno)); - } -#if defined NBD_SET_FLAGS && defined NBD_FLAG_SEND_TRIM - else if(ioctl(nbd, NBD_SET_FLAGS, NBD_FLAG_SEND_TRIM) == -1){ - fprintf(stderr, "ioctl(nbd, NBD_SET_FLAGS, NBD_FLAG_SEND_TRIM) failed.[%s]\n", strerror(errno)); - } -#endif - else{ - err = ioctl(nbd, NBD_DO_IT); - fprintf(stderr, "nbd device terminated with code %d\n", err); - if (err == -1) - fprintf(stderr, "%s\n", strerror(errno)); - } - - ioctl(nbd, NBD_CLEAR_QUE); - ioctl(nbd, NBD_CLEAR_SOCK); - - exit(0); - } - - /* The parent opens the device file at least once, to make sure the - * partition table is updated. Then it closes it and starts serving up - * requests. */ - - tmp_fd = open(dev_file, O_RDONLY); - assert(tmp_fd != -1); - close(tmp_fd); - - close(sp[1]); - sk = sp[0]; - reply.magic = htonl(NBD_REPLY_MAGIC); reply.error = htonl(0); @@ -175,7 +141,7 @@ int buse_main(const char* dev_file, const struct buse_operations *aop, void *use * and writes. */ case NBD_CMD_READ: - fprintf(stderr, "Request for read of size %d\n", len); + if (BUSE_DEBUG) fprintf(stderr, "Request for read of size %d\n", len); /* Fill with zero in case actual read is not implemented */ chunk = malloc(len); if (aop->read) { @@ -190,7 +156,7 @@ int buse_main(const char* dev_file, const struct buse_operations *aop, void *use free(chunk); break; case NBD_CMD_WRITE: - fprintf(stderr, "Request for write of size %d\n", len); + if (BUSE_DEBUG) fprintf(stderr, "Request for write of size %d\n", len); chunk = malloc(len); read_all(sk, chunk, len); if (aop->write) { @@ -203,13 +169,15 @@ int buse_main(const char* dev_file, const struct buse_operations *aop, void *use write_all(sk, (char*)&reply, sizeof(struct nbd_reply)); break; case NBD_CMD_DISC: + if (BUSE_DEBUG) fprintf(stderr, "Got NBD_CMD_DISC\n"); /* Handle a disconnect request. */ if (aop->disc) { aop->disc(userdata); } - return 0; + return EXIT_SUCCESS; #ifdef NBD_FLAG_SEND_FLUSH case NBD_CMD_FLUSH: + if (BUSE_DEBUG) fprintf(stderr, "Got NBD_CMD_FLUSH\n"); if (aop->flush) { reply.error = aop->flush(userdata); } @@ -218,6 +186,7 @@ int buse_main(const char* dev_file, const struct buse_operations *aop, void *use #endif #ifdef NBD_FLAG_SEND_TRIM case NBD_CMD_TRIM: + if (BUSE_DEBUG) fprintf(stderr, "Got NBD_CMD_TRIM\n"); if (aop->trim) { reply.error = aop->trim(from, len, userdata); } @@ -228,7 +197,138 @@ int buse_main(const char* dev_file, const struct buse_operations *aop, void *use assert(0); } } - if (bytes_read == -1) - fprintf(stderr, "%s\n", strerror(errno)); - return 0; + if (bytes_read == -1) { + warn("error reading userside of nbd socket"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int buse_main(const char* dev_file, const struct buse_operations *aop, void *userdata) +{ + int sp[2]; + int nbd, sk, err, flags; + + err = socketpair(AF_UNIX, SOCK_STREAM, 0, sp); + assert(!err); + + nbd = open(dev_file, O_RDWR); + if (nbd == -1) { + fprintf(stderr, + "Failed to open `%s': %s\n" + "Is kernel module `nbd' loaded and you have permissions " + "to access the device?\n", dev_file, strerror(errno)); + return 1; + } + + if (aop->blksize) { + err = ioctl(nbd, NBD_SET_BLKSIZE, aop->blksize); + assert(err != -1); + } + if (aop->size) { + err = ioctl(nbd, NBD_SET_SIZE, aop->size); + assert(err != -1); + } + if (aop->size_blocks) { + err = ioctl(nbd, NBD_SET_SIZE_BLOCKS, aop->size_blocks); + assert(err != -1); + } + + err = ioctl(nbd, NBD_CLEAR_SOCK); + assert(err != -1); + + pid_t pid = fork(); + if (pid == 0) { + /* Block all signals to not get interrupted in ioctl(NBD_DO_IT), as + * it seems there is no good way to handle such interruption.*/ + sigset_t sigset; + if ( + sigfillset(&sigset) != 0 || + sigprocmask(SIG_SETMASK, &sigset, NULL) != 0 + ) { + warn("failed to block signals in child"); + exit(EXIT_FAILURE); + } + + /* The child needs to continue setting things up. */ + close(sp[0]); + sk = sp[1]; + + if(ioctl(nbd, NBD_SET_SOCK, sk) == -1){ + fprintf(stderr, "ioctl(nbd, NBD_SET_SOCK, sk) failed.[%s]\n", strerror(errno)); + exit(EXIT_FAILURE); + } + else{ +#if defined NBD_SET_FLAGS + flags = 0; +#if defined NBD_FLAG_SEND_TRIM + flags |= NBD_FLAG_SEND_TRIM; +#endif +#if defined NBD_FLAG_SEND_FLUSH + flags |= NBD_FLAG_SEND_FLUSH; +#endif + if (flags != 0 && ioctl(nbd, NBD_SET_FLAGS, flags) == -1){ + fprintf(stderr, "ioctl(nbd, NBD_SET_FLAGS, %d) failed.[%s]\n", flags, strerror(errno)); + exit(EXIT_FAILURE); + } +#endif + err = ioctl(nbd, NBD_DO_IT); + if (BUSE_DEBUG) fprintf(stderr, "nbd device terminated with code %d\n", err); + if (err == -1) { + warn("NBD_DO_IT terminated with error"); + exit(EXIT_FAILURE); + } + } + + if ( + ioctl(nbd, NBD_CLEAR_QUE) == -1 || + ioctl(nbd, NBD_CLEAR_SOCK) == -1 + ) { + warn("failed to perform nbd cleanup actions"); + exit(EXIT_FAILURE); + } + + exit(0); + } + + /* Parent handles termination signals by terminating nbd device. */ + assert(nbd_dev_to_disconnect == -1); + nbd_dev_to_disconnect = nbd; + struct sigaction act; + act.sa_handler = disconnect_nbd; + act.sa_flags = SA_RESTART; + if ( + sigemptyset(&act.sa_mask) != 0 || + sigaddset(&act.sa_mask, SIGINT) != 0 || + sigaddset(&act.sa_mask, SIGTERM) != 0 + ) { + warn("failed to prepare signal mask in parent"); + return EXIT_FAILURE; + } + if ( + set_sigaction(SIGINT, &act) != 0 || + set_sigaction(SIGTERM, &act) != 0 + ) { + warn("failed to register signal handlers in parent"); + return EXIT_FAILURE; + } + + close(sp[1]); + + /* serve NBD socket */ + int status; + status = serve_nbd(sp[0], aop, userdata); + if (close(sp[0]) != 0) warn("problem closing server side nbd socket"); + if (status != 0) return status; + + /* wait for subprocess */ + if (waitpid(pid, &status, 0) == -1) { + warn("waitpid failed"); + return EXIT_FAILURE; + } + if (WEXITSTATUS(status) != 0) { + return WEXITSTATUS(status); + } + + return EXIT_SUCCESS; } diff --git a/buse.h b/buse.h index c3d379c..c3a08d2 100644 --- a/buse.h +++ b/buse.h @@ -4,11 +4,8 @@ #ifdef __cplusplus extern "C" { #endif - - /* Most of this file was copied from nbd.h in the nbd distribution. */ -#include + #include -#include struct buse_operations { int (*read)(void *buf, u_int32_t len, u_int64_t offset, void *userdata); diff --git a/busexmp.c b/busexmp.c index d342a6e..57cd503 100644 --- a/busexmp.c +++ b/busexmp.c @@ -17,14 +17,16 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ +#include +#include #include #include #include #include "buse.h" +/* BUSE callbacks */ static void *data; -static int xmpl_debug = 1; static int xmp_read(void *buf, u_int32_t len, u_int64_t offset, void *userdata) { @@ -44,47 +46,127 @@ static int xmp_write(const void *buf, u_int32_t len, u_int64_t offset, void *use static void xmp_disc(void *userdata) { - (void)(userdata); - fprintf(stderr, "Received a disconnect request.\n"); + if (*(int *)userdata) + fprintf(stderr, "Received a disconnect request.\n"); } static int xmp_flush(void *userdata) { - (void)(userdata); - fprintf(stderr, "Received a flush request.\n"); + if (*(int *)userdata) + fprintf(stderr, "Received a flush request.\n"); return 0; } static int xmp_trim(u_int64_t from, u_int32_t len, void *userdata) { - (void)(userdata); - fprintf(stderr, "T - %lu, %u\n", from, len); + if (*(int *)userdata) + fprintf(stderr, "T - %lu, %u\n", from, len); return 0; } +/* argument parsing using argp */ -static struct buse_operations aop = { - .read = xmp_read, - .write = xmp_write, - .disc = xmp_disc, - .flush = xmp_flush, - .trim = xmp_trim, - .size = 128 * 1024 * 1024, +static struct argp_option options[] = { + {"verbose", 'v', 0, 0, "Produce verbose output", 0}, + {0}, }; -int main(int argc, char *argv[]) -{ - if (argc != 2) - { - fprintf(stderr, - "Usage:\n" - " %s /dev/nbd0\n" - "Don't forget to load nbd kernel module (`modprobe nbd`) and\n" - "run example from root.\n", argv[0]); - return 1; +struct arguments { + unsigned long long size; + char * device; + int verbose; +}; + +static unsigned long long strtoull_with_prefix(const char * str, char * * end) { + unsigned long long v = strtoull(str, end, 0); + switch (**end) { + case 'K': + v *= 1024; + *end += 1; + break; + case 'M': + v *= 1024 * 1024; + *end += 1; + break; + case 'G': + v *= 1024 * 1024 * 1024; + *end += 1; + break; + } + return v; +} + +/* Parse a single option. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state) { + struct arguments *arguments = state->input; + char * endptr; + + switch (key) { + + case 'v': + arguments->verbose = 1; + break; + + case ARGP_KEY_ARG: + switch (state->arg_num) { + + case 0: + arguments->size = strtoull_with_prefix(arg, &endptr); + if (*endptr != '\0') { + /* failed to parse integer */ + errx(EXIT_FAILURE, "SIZE must be an integer"); + } + break; + + case 1: + arguments->device = arg; + break; + + default: + /* Too many arguments. */ + return ARGP_ERR_UNKNOWN; + } + break; + + case ARGP_KEY_END: + if (state->arg_num < 2) { + warnx("not enough arguments"); + argp_usage(state); + } + break; + + default: + return ARGP_ERR_UNKNOWN; } + return 0; +} + +static struct argp argp = { + .options = options, + .parser = parse_opt, + .args_doc = "SIZE DEVICE", + .doc = "BUSE virtual block device that stores its content in memory.\n" + "`SIZE` accepts suffixes K, M, G. `DEVICE` is path to block device, for example \"/dev/nbd0\".", +}; + + +int main(int argc, char *argv[]) { + struct arguments arguments = { + .verbose = 0, + }; + argp_parse(&argp, argc, argv, 0, 0, &arguments); + + struct buse_operations aop = { + .read = xmp_read, + .write = xmp_write, + .disc = xmp_disc, + .flush = xmp_flush, + .trim = xmp_trim, + .size = arguments.size, + }; data = malloc(aop.size); + if (data == NULL) err(EXIT_FAILURE, "failed to alloc space for data"); - return buse_main(argv[1], &aop, (void *)&xmpl_debug); + return buse_main(arguments.device, &aop, (void *)&arguments.verbose); } diff --git a/test/busexmp.sh b/test/busexmp.sh new file mode 100755 index 0000000..e72f846 --- /dev/null +++ b/test/busexmp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -e + +BLOCKDEV=/dev/nbd0 +# quiet version of dd +DD="dd status=none" + +# verify if blockdev is not currently in use +set +e +nbd-client -c "$BLOCKDEV" > /dev/null +if [ $? -ne 1 ]; then + echo "device $BLOCKDEV is not ready to use (already in use or corrupted)" + exit 1 +fi +set -e + +# on exit do cleanup actions +function cleanup () { + # kill and wait for BUSE background job + nbd-client -d "$BLOCKDEV" > /dev/null + wait $BUSEPID + # remove the test file + rm -f "$TESTFILE" +} +trap cleanup EXIT + +# prepare file with some data +TESTFILE=$(mktemp) +$DD if=/dev/urandom of="$TESTFILE" bs=16M count=1 + +# attach BUSE device +busexmp 128M "$BLOCKDEV" & +sleep 1 +BUSEPID=$! + +### do checks ### + +# initialy there are all zeros +cmp <($DD if="$BLOCKDEV" ibs=1k count=5 skip=54) <($DD if=/dev/zero bs=1k count=5) + +# write data at the end +$DD if="$TESTFILE" of="$BLOCKDEV" bs=1M count=2 seek=126 + +# read extending past the end of device +cmp <($DD if="$BLOCKDEV" bs=1M count=5 skip=126) <($DD if="$TESTFILE" bs=1M count=2) diff --git a/test/signal_termination.sh b/test/signal_termination.sh new file mode 100755 index 0000000..c14916e --- /dev/null +++ b/test/signal_termination.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e + +BLOCKDEV=/dev/nbd0 +SIZE=16M + +# verify if blockdev is not currently in use +set +e +nbd-client -c "$BLOCKDEV" > /dev/null +if [ $? -ne 1 ]; then + echo "device $BLOCKDEV is not ready to use (already in use or corrupted)" + exit 1 +fi +set -e + +# on exit make sure nbd is disconnected +function cleanup () { + nbd-client -d "$BLOCKDEV" > /dev/null +} +trap cleanup EXIT + +# attach BUSE device +busexmp "$SIZE" "$BLOCKDEV" & +BUSEPID=$! + +# wait a bit ensure buse is running and connected to device +sleep 1 +nbd-client -c "$BLOCKDEV" > /dev/null + +# kill it with SIGTERM. Exit code should be 0 - if not bash will break because we have -e option. +kill -s SIGTERM $BUSEPID +wait $BUSEPID + +# attach BUSE again to verify if device is left in usable state +busexmp "$SIZE" "$BLOCKDEV" & +BUSEPID=$! +sleep 1 +nbd-client -c "$BLOCKDEV" > /dev/null +kill -s SIGINT $BUSEPID # this time kill it with SIGINT +wait $BUSEPID