Browse Source

import: add new minimal tool "systemd-import" for pulling down foreign containers and install them locally

This adds a simply but powerful tool for downloading container images
from the most popular container solution used today. Use it like
this:

       # systemd-import pull-dck mattdm/fedora
       # systemd-nspawn -M fedora

This will donwload the layers for "mattdm/fedora", and make them
available locally as /var/lib/container/fedora.

The tool is pretty complete, as long as it's only about pulling down
images, or updating them. Pushing or searching is not supported yet.
keep-around/964a6d9fb555cc86528eb1cc1f6d044f85584842
Lennart Poettering 8 years ago
parent
commit
72648326ea
  1. 1
      .gitignore
  2. 26
      Makefile.am
  3. 1
      src/import/Makefile
  4. 73
      src/import/aufs-util.c
  5. 24
      src/import/aufs-util.h
  6. 415
      src/import/curl-util.c
  7. 56
      src/import/curl-util.h
  8. 1156
      src/import/import-dck.c
  9. 39
      src/import/import-dck.h
  10. 213
      src/import/import.c
  11. 2
      src/shared/util.h

1
.gitignore

@ -82,6 +82,7 @@
/systemd-hibernate-resume-generator
/systemd-hostnamed
/systemd-hwdb
/systemd-import
/systemd-inhibit
/systemd-initctl
/systemd-journal-gatewayd

26
Makefile.am

@ -5094,6 +5094,32 @@ libnss_mymachines_la_LIBADD = \
lib_LTLIBRARIES += \
libnss_mymachines.la
if HAVE_LIBCURL
bin_PROGRAMS += \
systemd-import
systemd_import_SOURCES = \
src/import/import.c \
src/import/import-dck.c \
src/import/import-dck.h \
src/import/curl-util.c \
src/import/curl-util.h \
src/import/aufs-util.c \
src/import/aufs-util.h
systemd_import_CFLAGS = \
$(AM_CFLAGS) \
$(LIBCURL_CFLAGS)
systemd_import_LDADD = \
libsystemd-internal.la \
libsystemd-shared.la \
$(LIBCURL_LIBS) \
-lm
endif
endif
# ------------------------------------------------------------------------------

1
src/import/Makefile

@ -0,0 +1 @@
../Makefile

73
src/import/aufs-util.c

@ -0,0 +1,73 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <ftw.h>
#include "util.h"
#include "aufs-util.h"
static int nftw_cb(
const char *fpath,
const struct stat *sb,
int flag,
struct FTW *ftwbuf) {
const char *fn, *original;
char *p;
int r;
fn = fpath + ftwbuf->base;
/* We remove all whiteout files, and all whiteouts */
original = startswith(fn, ".wh.");
if (!original)
return FTW_CONTINUE;
log_debug("Removing whiteout indicator %s.", fpath);
r = rm_rf_dangerous(fpath, false, true, false);
if (r < 0)
return FTW_STOP;
if (!startswith(fn, ".wh..wh.")) {
p = alloca(ftwbuf->base + strlen(original));
strcpy(mempcpy(p, fpath, ftwbuf->base), original);
log_debug("Removing deleted file %s.", p);
r = rm_rf_dangerous(p, false, true, false);
if (r < 0)
return FTW_STOP;
}
return FTW_CONTINUE;
}
int aufs_resolve(const char *path) {
int r;
errno = 0;
r = nftw(path, nftw_cb, 64, FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL);
if (r == FTW_STOP)
return errno ? -errno : -EIO;
return 0;
}

24
src/import/aufs-util.h

@ -0,0 +1,24 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
#pragma once
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
int aufs_resolve(const char *path);

415
src/import/curl-util.c

@ -0,0 +1,415 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include "curl-util.h"
static void curl_glue_check_finished(CurlGlue *g) {
CURLMsg *msg;
int k = 0;
assert(g);
msg = curl_multi_info_read(g->curl, &k);
if (!msg)
return;
if (msg->msg != CURLMSG_DONE)
return;
if (g->on_finished)
g->on_finished(g, msg->easy_handle, msg->data.result);
}
static int curl_glue_on_io(sd_event_source *s, int fd, uint32_t revents, void *userdata) {
CurlGlue *g = userdata;
int action, k = 0, translated_fd;
assert(s);
assert(g);
translated_fd = PTR_TO_INT(hashmap_get(g->translate_fds, INT_TO_PTR(fd+1)));
assert(translated_fd > 0);
translated_fd--;
if ((revents & (EPOLLIN|EPOLLOUT)) == (EPOLLIN|EPOLLOUT))
action = CURL_POLL_INOUT;
else if (revents & EPOLLIN)
action = CURL_POLL_IN;
else if (revents & EPOLLOUT)
action = CURL_POLL_OUT;
else
action = 0;
if (curl_multi_socket_action(g->curl, translated_fd, action, &k) < 0) {
log_debug("Failed to propagate IO event.");
return -EINVAL;
}
curl_glue_check_finished(g);
return 0;
}
static int curl_glue_socket_callback(CURLM *curl, curl_socket_t s, int action, void *userdata, void *socketp) {
sd_event_source *io;
CurlGlue *g = userdata;
uint32_t events = 0;
int r;
assert(curl);
assert(g);
io = hashmap_get(g->ios, INT_TO_PTR(s+1));
if (action == CURL_POLL_REMOVE) {
if (io) {
int fd;
fd = sd_event_source_get_io_fd(io);
assert(fd >= 0);
sd_event_source_set_enabled(io, SD_EVENT_OFF);
sd_event_source_unref(io);
hashmap_remove(g->ios, INT_TO_PTR(s+1));
hashmap_remove(g->translate_fds, INT_TO_PTR(fd+1));
safe_close(fd);
}
return 0;
}
r = hashmap_ensure_allocated(&g->ios, &trivial_hash_ops);
if (r < 0) {
log_oom();
return -1;
}
r = hashmap_ensure_allocated(&g->translate_fds, &trivial_hash_ops);
if (r < 0) {
log_oom();
return -1;
}
if (action == CURL_POLL_IN)
events = EPOLLIN;
else if (action == CURL_POLL_OUT)
events = EPOLLOUT;
else if (action == CURL_POLL_INOUT)
events = EPOLLIN|EPOLLOUT;
if (io) {
if (sd_event_source_set_io_events(io, events) < 0)
return -1;
if (sd_event_source_set_enabled(io, SD_EVENT_ON) < 0)
return -1;
} else {
_cleanup_close_ int fd = -1;
/* When curl needs to remove an fd from us it closes
* the fd first, and only then calls into us. This is
* nasty, since we cannot pass the fd on to epoll()
* anymore. Hence, duplicate the fds here, and keep a
* copy for epoll which we control after use. */
fd = fcntl(s, F_DUPFD_CLOEXEC, 3);
if (fd < 0)
return -1;
if (sd_event_add_io(g->event, &io, fd, events, curl_glue_on_io, g) < 0)
return -1;
sd_event_source_set_description(io, "curl-io");
r = hashmap_put(g->ios, INT_TO_PTR(s+1), io);
if (r < 0) {
log_oom();
sd_event_source_unref(io);
return -1;
}
r = hashmap_put(g->translate_fds, INT_TO_PTR(fd+1), INT_TO_PTR(s+1));
if (r < 0) {
log_oom();
hashmap_remove(g->ios, INT_TO_PTR(s+1));
sd_event_source_unref(io);
return -1;
}
fd = -1;
}
return 0;
}
static int curl_glue_on_timer(sd_event_source *s, uint64_t usec, void *userdata) {
CurlGlue *g = userdata;
int k = 0;
assert(s);
assert(g);
if (curl_multi_socket_action(g->curl, CURL_SOCKET_TIMEOUT, 0, &k) != CURLM_OK) {
log_debug("Failed to propagate timeout.");
return -EINVAL;
}
curl_glue_check_finished(g);
return 0;
}
static int curl_glue_timer_callback(CURLM *curl, long timeout_ms, void *userdata) {
CurlGlue *g = userdata;
usec_t usec;
assert(curl);
assert(g);
if (timeout_ms < 0) {
if (g->timer) {
if (sd_event_source_set_enabled(g->timer, SD_EVENT_OFF) < 0)
return -1;
}
return 0;
}
usec = now(clock_boottime_or_monotonic()) + (usec_t) timeout_ms * USEC_PER_MSEC + USEC_PER_MSEC - 1;
if (g->timer) {
if (sd_event_source_set_time(g->timer, usec) < 0)
return -1;
if (sd_event_source_set_enabled(g->timer, SD_EVENT_ONESHOT) < 0)
return -1;
} else {
if (sd_event_add_time(g->event, &g->timer, clock_boottime_or_monotonic(), usec, 0, curl_glue_on_timer, g) < 0)
return -1;
sd_event_source_set_description(g->timer, "curl-timer");
}
return 0;
}
CurlGlue *curl_glue_unref(CurlGlue *g) {
sd_event_source *io;
if (!g)
return NULL;
if (g->curl)
curl_multi_cleanup(g->curl);
while ((io = hashmap_steal_first(g->ios))) {
int fd;
fd = sd_event_source_get_io_fd(io);
assert(fd >= 0);
hashmap_remove(g->translate_fds, INT_TO_PTR(fd+1));
safe_close(fd);
sd_event_source_unref(io);
}
hashmap_free(g->ios);
sd_event_source_unref(g->timer);
sd_event_unref(g->event);
return NULL;
}
int curl_glue_new(CurlGlue **glue, sd_event *event) {
_cleanup_(curl_glue_unrefp) CurlGlue *g = NULL;
int r;
g = new0(CurlGlue, 1);
if (!g)
return -ENOMEM;
if (event)
g->event = sd_event_ref(event);
else {
r = sd_event_default(&g->event);
if (r < 0)
return r;
}
g->curl = curl_multi_init();
if (!g->curl)
return -ENOMEM;
if (curl_multi_setopt(g->curl, CURLMOPT_SOCKETDATA, g) != CURLM_OK)
return -EINVAL;
if (curl_multi_setopt(g->curl, CURLMOPT_SOCKETFUNCTION, curl_glue_socket_callback) != CURLM_OK)
return -EINVAL;
if (curl_multi_setopt(g->curl, CURLMOPT_TIMERDATA, g) != CURLM_OK)
return -EINVAL;
if (curl_multi_setopt(g->curl, CURLMOPT_TIMERFUNCTION, curl_glue_timer_callback) != CURLM_OK)
return -EINVAL;
*glue = g;
g = NULL;
return 0;
}
int curl_glue_make(CURL **ret, const char *url, void *userdata) {
const char *useragent;
CURL *c;
int r;
assert(ret);
assert(url);
c = curl_easy_init();
if (!c)
return -ENOMEM;
/* curl_easy_setopt(c, CURLOPT_VERBOSE, 1L); */
if (curl_easy_setopt(c, CURLOPT_URL, url) != CURLE_OK) {
r = -EIO;
goto fail;
}
if (curl_easy_setopt(c, CURLOPT_PRIVATE, userdata) != CURLE_OK) {
r = -EIO;
goto fail;
}
useragent = strappenda(program_invocation_short_name, "/" PACKAGE_VERSION);
if (curl_easy_setopt(c, CURLOPT_USERAGENT, useragent) != CURLE_OK) {
r = -EIO;
goto fail;
}
if (curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) {
r = -EIO;
goto fail;
}
*ret = c;
return 0;
fail:
curl_easy_cleanup(c);
return r;
}
int curl_glue_add(CurlGlue *g, CURL *c) {
assert(g);
assert(c);
if (curl_multi_add_handle(g->curl, c) != CURLM_OK)
return -EIO;
return 0;
}
void curl_glue_remove_and_free(CurlGlue *g, CURL *c) {
assert(g);
if (!c)
return;
if (g->curl)
curl_multi_remove_handle(g->curl, c);
curl_easy_cleanup(c);
}
struct curl_slist *curl_slist_new(const char *first, ...) {
struct curl_slist *l;
va_list ap;
if (!first)
return NULL;
l = curl_slist_append(NULL, first);
if (!l)
return NULL;
va_start(ap, first);
for (;;) {
struct curl_slist *n;
const char *i;
i = va_arg(ap, const char*);
if (!i)
break;
n = curl_slist_append(l, i);
if (!n) {
va_end(ap);
curl_slist_free_all(l);
return NULL;
}
l = n;
}
va_end(ap);
return l;
}
int curl_header_strdup(const void *contents, size_t sz, const char *field, char **value) {
const char *p = contents;
size_t l;
char *s;
l = strlen(field);
if (sz < l)
return 0;
if (memcmp(p, field, l) != 0)
return 0;
p += l;
sz -= l;
if (memchr(p, 0, sz))
return 0;
/* Skip over preceeding whitespace */
while (sz > 0 && strchr(WHITESPACE, p[0])) {
p++;
sz--;
}
/* Truncate trailing whitespace*/
while (sz > 0 && strchr(WHITESPACE, p[sz-1]))
sz--;
s = strndup(p, sz);
if (!s)
return -ENOMEM;
*value = s;
return 1;
}

56
src/import/curl-util.h

@ -0,0 +1,56 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
#pragma once
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <sys/types.h>
#include <curl/curl.h>
#include "hashmap.h"
#include "sd-event.h"
typedef struct CurlGlue CurlGlue;
struct CurlGlue {
sd_event *event;
CURLM *curl;
sd_event_source *timer;
Hashmap *ios;
Hashmap *translate_fds;
void (*on_finished)(CurlGlue *g, CURL *curl, CURLcode code);
void *userdata;
};
int curl_glue_new(CurlGlue **glue, sd_event *event);
CurlGlue* curl_glue_unref(CurlGlue *glue);
DEFINE_TRIVIAL_CLEANUP_FUNC(CurlGlue*, curl_glue_unref);
int curl_glue_make(CURL **ret, const char *url, void *userdata);
int curl_glue_add(CurlGlue *g, CURL *c);
void curl_glue_remove_and_free(CurlGlue *g, CURL *c);
struct curl_slist *curl_slist_new(const char *first, ...) _sentinel_;
int curl_header_strdup(const void *contents, size_t sz, const char *field, char **value);
DEFINE_TRIVIAL_CLEANUP_FUNC(CURL*, curl_easy_cleanup);
DEFINE_TRIVIAL_CLEANUP_FUNC(struct curl_slist*, curl_slist_free_all);

1156
src/import/import-dck.c

File diff suppressed because it is too large

39
src/import/import-dck.h

@ -0,0 +1,39 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include "sd-event.h"
#include "util.h"
typedef struct DckImport DckImport;
typedef void (*dck_import_on_finished)(DckImport *import, int error, void *userdata);
int dck_import_new(DckImport **import, sd_event *event, dck_import_on_finished on_finished, void *userdata);
DckImport* dck_import_unref(DckImport *import);
DEFINE_TRIVIAL_CLEANUP_FUNC(DckImport*, dck_import_unref);
int dck_import_pull(DckImport *import, const char *name, const char *tag, const char *local, bool force_local);
int dck_import_cancel(DckImport *import, const char *name);
bool dck_name_is_valid(const char *name);
bool dck_id_is_valid(const char *id);
#define dck_tag_is_valid(tag) filename_is_valid(tag)

213
src/import/import.c

@ -0,0 +1,213 @@
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
/***
This file is part of systemd.
Copyright 2014 Lennart Poettering
systemd is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
systemd is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with systemd; If not, see <http://www.gnu.org/licenses/>.
***/
#include <getopt.h>
#include "sd-event.h"
#include "event-util.h"
#include "verbs.h"
#include "build.h"
#include "import-dck.h"
static bool arg_force = false;
static void on_finished(DckImport *import, int error, void *userdata) {
sd_event *event = userdata;
assert(import);
if (error == 0)
log_info("Operation completed successfully.");
else
log_info_errno(error, "Operation failed: %m");
sd_event_exit(event, error);
}
static int pull_dck(int argc, char *argv[], void *userdata) {
_cleanup_(dck_import_unrefp) DckImport *import = NULL;
_cleanup_event_unref_ sd_event *event = NULL;
const char *name, *tag, *local;
int r;
tag = strchr(argv[1], ':');
if (tag) {
name = strndupa(argv[1], tag - argv[1]);
tag++;
} else {
name = argv[1];
tag = "latest";
}
if (argc >= 3)
local = argv[2];
else {
local = strchr(name, '/');
if (local)
local++;
else
local = name;
}
if (streq(local, "-") || isempty(local))
local = NULL;
if (!dck_name_is_valid(name)) {
log_error("Remote name '%s' is not valid.", name);
return -EINVAL;
}
if (!dck_tag_is_valid(tag)) {
log_error("Tag name '%s' is not valid.", tag);
return -EINVAL;
}
if (local) {
const char *p;
if (!machine_name_is_valid(tag)) {
log_error("Local image name '%s' is not valid.", local);
return -EINVAL;
}
p = strappenda("/var/lib/container/", local);
if (laccess(p, F_OK) >= 0) {
if (!arg_force) {
log_info("Image '%s' already exists.", local);
return 0;
}
} else if (errno != ENOENT)
return log_error_errno(errno, "Can't check if image '%s' already exists: %m", local);
log_info("Pulling '%s' with tag '%s', saving as '%s'.", name, tag, local);
} else
log_info("Pulling '%s' with tag '%s'.", name, tag);
r = sd_event_default(&event);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
assert_se(sigprocmask_many(SIG_BLOCK, SIGTERM, SIGINT, -1) == 0);
sd_event_add_signal(event, NULL, SIGTERM, NULL, NULL);
sd_event_add_signal(event, NULL, SIGINT, NULL, NULL);
r = dck_import_new(&import, event, on_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate importer: %m");
r = dck_import_pull(import, name, tag, local, arg_force);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
r = sd_event_loop(event);
if (r < 0)
return log_error_errno(r, "Failed to run event loop: %m");
log_info("Exiting.");
return 0;
}
static int help(int argc, char *argv[], void *userdata) {
printf("%s [OPTIONS...] {COMMAND} ...\n\n"
"Import container or virtual machine image.\n\n"
" -h --help Show this help\n"
" --version Show package version\n"
" --force Force creation of image\n\n"
"Commands:\n"
" pull-dck REMOTE [NAME] Download an image\n",
program_invocation_short_name);
return 0;
}
static int parse_argv(int argc, char *argv[]) {
enum {
ARG_VERSION = 0x100,
ARG_FORCE,
};
static const struct option options[] = {
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, ARG_VERSION },
{ "force", no_argument, NULL, ARG_FORCE },
{}
};
int c;
assert(argc >= 0);
assert(argv);
while ((c = getopt_long(argc, argv, "h", options, NULL)) >= 0)
switch (c) {
case 'h':
return help(argc, argv, NULL);
case ARG_VERSION:
puts(PACKAGE_STRING);
puts(SYSTEMD_FEATURES);
return 0;
case ARG_FORCE:
arg_force = true;
break;
case '?':
return -EINVAL;
default:
assert_not_reached("Unhandled option");
}
return 1;
}
static int import_main(int argc, char *argv[]) {
const Verb verbs[] = {
{ "help", VERB_ANY, VERB_ANY, 0, help },
{ "pull-dck", 2, 3, 0, pull_dck },
{}
};
return dispatch_verb(argc, argv, verbs, NULL);
}
int main(int argc, char *argv[]) {
int r;
setlocale(LC_ALL, "");
log_parse_environment();
log_open();
r = parse_argv(argc, argv);
if (r <= 0)
goto finish;
r = import_main(argc, argv);
finish:
return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
}

2
src/shared/util.h

@ -1040,3 +1040,5 @@ int sethostname_idempotent(const char *s);
for ((e) = (struct inotify_event*) (buffer); \
(uint8_t*) (e) < (uint8_t*) (buffer) + (sz); \
(e) = (struct inotify_event*) ((uint8_t*) (e) + sizeof(struct inotify_event) + (e)->len))
#define laccess(path, mode) faccessat(AT_FDCWD, (path), (mode), AT_SYMLINK_NOFOLLOW)

Loading…
Cancel
Save