From 27041ccc22eed8fb94a0b389c1b186234174d396 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Tue, 5 Nov 2013 10:21:32 +0000
Subject: [PATCH] builder: Internal implementation of parallel xzcat (pxzcat).
Instead of calling out to the pxzcat program, use an internal
implementation. This requires liblzma to be available at build time.
If it's not available, fall back to using regular xzcat.
It is intended that eventually this code will go away when regular
xzcat / unxz is able to use threads.
(cherry picked from commit 681c88ef5d69b1dec4b929ff3451c45968023e5a)
---
README | 2 +-
builder/Makefile.am | 11 +-
builder/builder.ml | 24 +-
builder/pxzcat-c.c | 650 +++++++++++++++++++++++++++++++++++++++++++++++
builder/pxzcat.ml | 19 ++
builder/pxzcat.mli | 31 +++
builder/virt-builder.pod | 7 +-
configure.ac | 10 +-
mllib/config.ml.in | 6 -
po/POTFILES | 1 +
po/POTFILES-ml | 1 +
11 files changed, 726 insertions(+), 36 deletions(-)
create mode 100644 builder/pxzcat-c.c
create mode 100644 builder/pxzcat.ml
create mode 100644 builder/pxzcat.mli
diff --git a/README b/README
index 461915d..baf4de4 100644
--- a/README
+++ b/README
@@ -174,7 +174,7 @@ The full requirements are described below.
| gpg | | O | Used by virt-builder for digital |
| | | | signatures |
+--------------+-------------+---+-----------------------------------------+
-| pxzcat | | O | Can be used by virt-builder for fast |
+| liblzma | | O | Can be used by virt-builder for fast |
| | | | uncompression of templates. |
+--------------+-------------+---+-----------------------------------------+
| findlib | | O | For the OCaml bindings. |
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 0af3f95..9430d62 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -21,7 +21,9 @@ AM_YFLAGS = -d
AM_CFLAGS = \
-I$(shell $(OCAMLC) -where) \
-I$(top_srcdir)/src \
- -I$(top_srcdir)/fish
+ -I$(top_srcdir)/fish \
+ -pthread \
+ $(LIBLZMA_CFLAGS)
EXTRA_DIST = \
$(SOURCES) \
@@ -47,6 +49,9 @@ SOURCES = \
list_entries.ml \
perl_edit.ml \
perl_edit.mli \
+ pxzcat.ml \
+ pxzcat.mli \
+ pxzcat-c.c \
sigchecker.mli \
sigchecker.ml
@@ -74,6 +79,8 @@ OBJECTS = \
index-struct.o \
index-parse.o \
index-parser-c.o \
+ pxzcat-c.o \
+ pxzcat.cmx \
get_kernel.cmx \
downloader.cmx \
sigchecker.cmx \
@@ -103,7 +110,7 @@ OCAMLOPTFLAGS = $(OCAMLCFLAGS)
virt-builder: $(OBJECTS)
$(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) \
mlguestfs.cmxa -linkpkg $^ \
- -cclib '-lncurses -lcrypt' \
+ -cclib '-pthread $(LIBLZMA_LIBS) -lncurses -lcrypt -lpthread' \
$(OCAML_GCOV_LDFLAGS) \
-o $@
diff --git a/builder/builder.ml b/builder/builder.ml
index 6a4ef70..4f000fd 100644
--- a/builder/builder.ml
+++ b/builder/builder.ml
@@ -24,6 +24,7 @@ open Common_utils
open Password
open Cmdline
+open Pxzcat
open Unix
open Printf
@@ -333,27 +334,14 @@ let main () =
output, Some size, format, delete_output_file, do_resize, true in
- (* Create xzcat/pxzcat command to uncompress from input to output. *)
- let xzcat_command input output =
- match Config.pxzcat with
- | None -> sprintf "%s %s > %s" Config.xzcat input output
- | Some pxzcat -> sprintf "%s %s -o %s" pxzcat input output
- in
-
if not do_resize then (
(* If the user did not specify --size and the output is a regular
* file and the format is raw, then we just uncompress the template
* directly to the output file. This is fast but less flexible.
*)
let { Index_parser.file_uri = file_uri } = entry in
- let cmd = xzcat_command template output in
- if debug then eprintf "%s\n%!" cmd;
msg (f_"Uncompressing: %s") file_uri;
- let r = Sys.command cmd in
- if r <> 0 then (
- eprintf (f_"%s: error: failed to uncompress template\n") prog;
- exit 1
- )
+ pxzcat template output
) else (
(* If none of the above apply, uncompress to a temporary file and
* run virt-resize on the result.
@@ -362,14 +350,8 @@ let main () =
(* Uncompress it to a temporary file. *)
let { Index_parser.file_uri = file_uri } = entry in
let tmpfile = Filename.temp_file "vbsrc" ".img" in
- let cmd = xzcat_command template tmpfile in
- if debug then eprintf "%s\n%!" cmd;
msg (f_"Uncompressing: %s") file_uri;
- let r = Sys.command cmd in
- if r <> 0 then (
- eprintf (f_"%s: error: failed to uncompress template\n") prog;
- exit 1
- );
+ pxzcat template tmpfile;
unlink_on_exit tmpfile;
tmpfile in
diff --git a/builder/pxzcat-c.c b/builder/pxzcat-c.c
new file mode 100644
index 0000000..5ffc9d9
--- /dev/null
+++ b/builder/pxzcat-c.c
@@ -0,0 +1,650 @@
+/* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <pthread.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+#ifdef HAVE_CAML_UNIXSUPPORT_H
+#include <caml/unixsupport.h>
+#else
+#define Nothing ((value) 0)
+extern void unix_error (int errcode, char * cmdname, value arg) Noreturn;
+#endif
+
+#ifdef HAVE_LIBLZMA
+#include <lzma.h>
+
+static void pxzcat (value filenamev, value outputfilev, unsigned nr_threads);
+#endif
+
+value
+virt_builder_pxzcat (value inputfilev, value outputfilev)
+{
+ CAMLparam2 (inputfilev, outputfilev);
+
+#ifdef HAVE_LIBLZMA
+
+ /* Parallel implementation of xzcat (pxzcat). */
+ /* XXX Make number of threads configurable? */
+ long i;
+ unsigned nr_threads;
+
+ i = sysconf (_SC_NPROCESSORS_ONLN);
+ if (i <= 0) {
+ perror ("could not get number of cores");
+ i = 1;
+ }
+ nr_threads = (unsigned) i;
+
+ /* NB: This might throw an exception if something fails. If it
+ * does, this function won't return as a regular C function.
+ */
+ pxzcat (inputfilev, outputfilev, nr_threads);
+
+#else
+
+ /* Fallback: use regular xzcat. */
+ int fd;
+ pid_t pid;
+ int status;
+
+ fd = open (String_val (outputfilev), O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, 0666);
+ if (fd == -1)
+ unix_error (errno, "open", outputfilev);
+
+ pid = fork ();
+ if (pid == -1) {
+ int err = errno;
+ close (fd);
+ unix_error (err, "fork", Nothing);
+ }
+
+ if (pid == 0) { /* child - run xzcat */
+ dup2 (fd, 1);
+ execlp (XZCAT, XZCAT, String_val (inputfilev), NULL);
+ perror (XZCAT);
+ _exit (EXIT_FAILURE);
+ }
+
+ close (fd);
+
+ if (waitpid (pid, &status, 0) == -1)
+ unix_error (errno, "waitpid", Nothing);
+ if (!WIFEXITED (status) || WEXITSTATUS (status) != 0)
+ caml_failwith (XZCAT " program failed, see earlier error messages");
+
+#endif
+
+ CAMLreturn (Val_unit);
+}
+
+#ifdef HAVE_LIBLZMA
+
+#define DEBUG 0
+
+#if DEBUG
+#define debug(fs,...) fprintf (stderr, "pxzcat: debug: " fs "\n", ## __VA_ARGS__)
+#else
+#define debug(fs,...) /* nothing */
+#endif
+
+/* Size of buffers used in decompression loop. */
+#define BUFFER_SIZE (64*1024)
+
+#define XZ_HEADER_MAGIC "\xfd" "7zXZ\0"
+#define XZ_HEADER_MAGIC_LEN 6
+#define XZ_FOOTER_MAGIC "YZ"
+#define XZ_FOOTER_MAGIC_LEN 2
+
+static int check_header_magic (int fd);
+static lzma_index *parse_indexes (value filenamev, int fd);
+static void iter_blocks (lzma_index *idx, unsigned nr_threads, value filenamev, int fd, value outputfilev, int ofd);
+
+static void
+pxzcat (value filenamev, value outputfilev, unsigned nr_threads)
+{
+ int fd, ofd;
+ uint64_t size;
+ lzma_index *idx;
+
+ /* Open the file. */
+ fd = open (String_val (filenamev), O_RDONLY);
+ if (fd == -1)
+ unix_error (errno, "open", filenamev);
+
+ /* Check file magic. */
+ if (!check_header_magic (fd)) {
+ close (fd);
+ caml_invalid_argument ("input file is not an xz file");
+ }
+
+ /* Read and parse the indexes. */
+ idx = parse_indexes (filenamev, fd);
+
+ /* Get the file uncompressed size, create the output file. */
+ size = lzma_index_uncompressed_size (idx);
+ debug ("uncompressed size = %" PRIu64 " bytes", size);
+
+ /* Avoid annoying ext4 auto_da_alloc which causes a flush on close
+ * unless we are very careful about not truncating a regular file
+ * from non-zero size to zero size. (Thanks Eric Sandeen)
+ */
+ ofd = open (String_val (outputfilev), O_WRONLY|O_CREAT|O_NOCTTY, 0644);
+ if (ofd == -1) {
+ int err = errno;
+ close (fd);
+ unix_error (err, "open", outputfilev);
+ }
+
+ if (ftruncate (ofd, 1) == -1) {
+ int err = errno;
+ close (fd);
+ unix_error (err, "ftruncate", outputfilev);
+ }
+
+ if (lseek (ofd, 0, SEEK_SET) == -1) {
+ int err = errno;
+ close (fd);
+ unix_error (err, "lseek", outputfilev);
+ }
+
+ if (write (ofd, "\0", 1) == -1) {
+ int err = errno;
+ close (fd);
+ unix_error (err, "write", outputfilev);
+ }
+
+ if (ftruncate (ofd, size) == -1) {
+ int err = errno;
+ close (fd);
+ unix_error (err, "ftruncate", outputfilev);
+ }
+
+ /* Tell the kernel we won't read the output file. */
+ posix_fadvise (fd, 0, 0, POSIX_FADV_RANDOM|POSIX_FADV_DONTNEED);
+
+ /* Iterate over blocks. */
+ iter_blocks (idx, nr_threads, filenamev, fd, outputfilev, ofd);
+
+ lzma_index_end (idx, NULL);
+
+ if (close (fd) == -1)
+ unix_error (errno, "close", filenamev);
+}
+
+static int
+check_header_magic (int fd)
+{
+ char buf[XZ_HEADER_MAGIC_LEN];
+
+ if (lseek (fd, 0, SEEK_SET) == -1)
+ return 0;
+ if (read (fd, buf, XZ_HEADER_MAGIC_LEN) != XZ_HEADER_MAGIC_LEN)
+ return 0;
+ if (memcmp (buf, XZ_HEADER_MAGIC, XZ_HEADER_MAGIC_LEN) != 0)
+ return 0;
+ return 1;
+}
+
+/* For explanation of this function, see src/xz/list.c:parse_indexes
+ * in the xz sources.
+ */
+static lzma_index *
+parse_indexes (value filenamev, int fd)
+{
+ lzma_ret r;
+ off_t pos, index_size;
+ uint8_t footer[LZMA_STREAM_HEADER_SIZE];
+ uint8_t header[LZMA_STREAM_HEADER_SIZE];
+ lzma_stream_flags footer_flags;
+ lzma_stream_flags header_flags;
+ lzma_stream strm = LZMA_STREAM_INIT;
+ ssize_t n;
+ lzma_index *combined_index = NULL;
+ lzma_index *this_index = NULL;
+ lzma_vli stream_padding = 0;
+ size_t nr_streams = 0;
+
+ /* Check file size is a multiple of 4 bytes. */
+ pos = lseek (fd, 0, SEEK_END);
+ if (pos == (off_t) -1)
+ unix_error (errno, "lseek", filenamev);
+
+ if ((pos & 3) != 0)
+ caml_invalid_argument ("input not an xz file: size is not a multiple of 4 bytes");
+
+ /* Jump backwards through the file identifying each stream. */
+ while (pos > 0) {
+ debug ("looping through streams: pos = %" PRIu64, (uint64_t) pos);
+
+ if (pos < LZMA_STREAM_HEADER_SIZE)
+ caml_invalid_argument ("corrupted xz file");
+
+ if (lseek (fd, -LZMA_STREAM_HEADER_SIZE, SEEK_CUR) == -1)
+ unix_error (errno, "lseek", filenamev);
+
+ if (read (fd, footer, LZMA_STREAM_HEADER_SIZE) != LZMA_STREAM_HEADER_SIZE)
+ unix_error (errno, "read", filenamev);
+
+ /* Skip stream padding. */
+ if (footer[8] == 0 && footer[9] == 0 &&
+ footer[10] == 0 && footer[11] == 0) {
+ stream_padding += 4;
+ pos -= 4;
+ continue;
+ }
+
+ pos -= LZMA_STREAM_HEADER_SIZE;
+ nr_streams++;
+
+ debug ("decode stream footer at pos = %" PRIu64, (uint64_t) pos);
+
+ /* Does the stream footer look reasonable? */
+ r = lzma_stream_footer_decode (&footer_flags, footer);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "invalid stream footer - error %d\n", r);
+ caml_invalid_argument ("invalid stream footer");
+ }
+
+ debug ("backward_size = %" PRIu64, (uint64_t) footer_flags.backward_size);
+ index_size = footer_flags.backward_size;
+ if (pos < index_size + LZMA_STREAM_HEADER_SIZE)
+ caml_invalid_argument ("invalid stream footer");
+
+ pos -= index_size;
+ debug ("decode index at pos = %" PRIu64, (uint64_t) pos);
+
+ /* Seek backwards to the index of this stream. */
+ if (lseek (fd, pos, SEEK_SET) == -1)
+ unix_error (errno, "lseek", filenamev);
+
+ /* Decode the index. */
+ r = lzma_index_decoder (&strm, &this_index, UINT64_MAX);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "invalid stream index - error %d\n", r);
+ caml_invalid_argument ("invalid stream index");
+ }
+
+ do {
+ uint8_t buf[BUFSIZ];
+
+ strm.avail_in = index_size;
+ if (strm.avail_in > BUFSIZ)
+ strm.avail_in = BUFSIZ;
+
+ n = read (fd, &buf, strm.avail_in);
+ if (n == -1)
+ unix_error (errno, "read", filenamev);
+
+ index_size -= strm.avail_in;
+
+ strm.next_in = buf;
+ r = lzma_code (&strm, LZMA_RUN);
+ } while (r == LZMA_OK);
+
+ if (r != LZMA_STREAM_END) {
+ fprintf (stderr, "could not parse index - error %d\n", r);
+ caml_invalid_argument ("could not parse index");
+ }
+
+ pos -= lzma_index_total_size (this_index) + LZMA_STREAM_HEADER_SIZE;
+
+ debug ("decode stream header at pos = %" PRIu64, (uint64_t) pos);
+
+ /* Read and decode the stream header. */
+ if (lseek (fd, pos, SEEK_SET) == -1)
+ unix_error (errno, "lseek", filenamev);
+
+ if (read (fd, header, LZMA_STREAM_HEADER_SIZE) != LZMA_STREAM_HEADER_SIZE)
+ unix_error (errno, "read stream header", filenamev);
+
+ r = lzma_stream_header_decode (&header_flags, header);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "invalid stream header - error %d\n", r);
+ caml_invalid_argument ("invalid stream header");
+ }
+
+ /* Header and footer of the stream should be equal. */
+ r = lzma_stream_flags_compare (&header_flags, &footer_flags);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "header and footer of stream are not equal - error %d\n",
+ r);
+ caml_invalid_argument ("header and footer of stream are not equal");
+ }
+
+ /* Store the decoded stream flags in this_index. */
+ r = lzma_index_stream_flags (this_index, &footer_flags);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "cannot read stream_flags from index - error %d\n", r);
+ caml_invalid_argument ("cannot read stream_flags from index");
+ }
+
+ /* Store the amount of stream padding so far. Needed to calculate
+ * compressed offsets correctly in multi-stream files.
+ */
+ r = lzma_index_stream_padding (this_index, stream_padding);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "cannot set stream_padding in index - error %d\n", r);
+ caml_invalid_argument ("cannot set stream_padding in index");
+ }
+
+ if (combined_index != NULL) {
+ r = lzma_index_cat (this_index, combined_index, NULL);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "cannot combine indexes - error %d\n", r);
+ caml_invalid_argument ("cannot combine indexes");
+ }
+ }
+
+ combined_index = this_index;
+ this_index = NULL;
+ }
+
+ lzma_end (&strm);
+
+ return combined_index;
+}
+
+/* Return true iff the buffer is all zero bytes.
+ *
+ * Note that gcc is smart enough to optimize this properly:
+ * http://stackoverflow.com/questions/1493936/faster-means-of-checking-for-an-empty-buffer-in-c/1493989#1493989
+ */
+static inline int
+is_zero (const char *buffer, size_t size)
+{
+ size_t i;
+
+ for (i = 0; i < size; ++i) {
+ if (buffer[i] != 0)
+ return 0;
+ }
+
+ return 1;
+}
+
+struct global_state {
+ /* Current iterator. Threads update this, but it is protected by a
+ * mutex, and each thread takes a copy of it when working on it.
+ */
+ lzma_index_iter iter;
+ lzma_bool iter_finished;
+ pthread_mutex_t iter_mutex;
+
+ /* Note that all threads are accessing these fds, so you have
+ * to use pread/pwrite instead of lseek!
+ */
+
+ /* Input file. */
+ const char *filename;
+ int fd;
+
+ /* Output file. */
+ const char *outputfile;
+ int ofd;
+};
+
+struct per_thread_state {
+ unsigned thread_num;
+ struct global_state *global;
+ int status;
+};
+
+/* Create threads to iterate over the blocks and uncompress. */
+static void *worker_thread (void *vp);
+
+static void
+iter_blocks (lzma_index *idx, unsigned nr_threads,
+ value filenamev, int fd, value outputfilev, int ofd)
+{
+ struct global_state global;
+ struct per_thread_state per_thread[nr_threads];
+ pthread_t thread[nr_threads];
+ unsigned u, nr_errors;
+ int err;
+ void *status;
+
+ lzma_index_iter_init (&global.iter, idx);
+ global.iter_finished = 0;
+ err = pthread_mutex_init (&global.iter_mutex, NULL);
+ if (err != 0)
+ unix_error (err, "pthread_mutex_init", Nothing);
+
+ global.filename = String_val (filenamev);
+ global.fd = fd;
+ global.outputfile = String_val (outputfilev);
+ global.ofd = ofd;
+
+ for (u = 0; u < nr_threads; ++u) {
+ per_thread[u].thread_num = u;
+ per_thread[u].global = &global;
+ }
+
+ /* Start the threads. */
+ for (u = 0; u < nr_threads; ++u) {
+ err = pthread_create (&thread[u], NULL, worker_thread, &per_thread[u]);
+ if (err != 0)
+ unix_error (err, "pthread_create", Nothing);
+ }
+
+ /* Wait for the threads to exit. */
+ nr_errors = 0;
+ for (u = 0; u < nr_threads; ++u) {
+ err = pthread_join (thread[u], &status);
+ if (err != 0) {
+ fprintf (stderr, "pthread_join (%u): %s\n", u, strerror (err));
+ nr_errors++;
+ }
+ if (*(int *)status == -1)
+ nr_errors++;
+ }
+
+ if (nr_errors > 0)
+ caml_invalid_argument ("some threads failed, see earlier errors");
+}
+
+/* Iterate over the blocks and uncompress. */
+static void *
+worker_thread (void *vp)
+{
+ struct per_thread_state *state = vp;
+ struct global_state *global = state->global;
+ lzma_index_iter iter;
+ int err;
+ off_t position, oposition;
+ uint8_t header[LZMA_BLOCK_HEADER_SIZE_MAX];
+ ssize_t n;
+ lzma_block block;
+ lzma_filter filters[LZMA_FILTERS_MAX + 1];
+ lzma_ret r;
+ lzma_stream strm = LZMA_STREAM_INIT;
+ uint8_t buf[BUFFER_SIZE];
+ char outbuf[BUFFER_SIZE];
+ size_t i;
+ lzma_bool iter_finished;
+
+ state->status = -1;
+
+ for (;;) {
+ /* Get the next block. */
+ err = pthread_mutex_lock (&global->iter_mutex);
+ if (err != 0) abort ();
+ iter_finished = global->iter_finished;
+ if (!iter_finished) {
+ iter_finished = global->iter_finished =
+ lzma_index_iter_next (&global->iter, LZMA_INDEX_ITER_NONEMPTY_BLOCK);
+ if (!iter_finished)
+ /* Take a local copy of this iterator since another thread will
+ * update the global version.
+ */
+ iter = global->iter;
+ }
+ err = pthread_mutex_unlock (&global->iter_mutex);
+ if (err != 0) abort ();
+ if (iter_finished)
+ break;
+
+ /* Read the block header. Start by reading a single byte which
+ * tell us how big the block header is.
+ */
+ position = iter.block.compressed_file_offset;
+ n = pread (global->fd, header, 1, position);
+ if (n == 0) {
+ fprintf (stderr,
+ "%s: read: unexpected end of file reading block header byte\n",
+ global->filename);
+ return &state->status;
+ }
+ if (n == -1) {
+ perror (String_val (global->filename));
+ return &state->status;
+ }
+ position++;
+
+ if (header[0] == '\0') {
+ fprintf (stderr,
+ "%s: read: unexpected invalid block in file, header[0] = 0\n",
+ global->filename);
+ return &state->status;
+ }
+
+ block.version = 0;
+ block.check = iter.stream.flags->check;
+ block.filters = filters;
+ block.header_size = lzma_block_header_size_decode (header[0]);
+
+ /* Now read and decode the block header. */
+ n = pread (global->fd, &header[1], block.header_size-1, position);
+ if (n >= 0 && n != block.header_size-1) {
+ fprintf (stderr,
+ "%s: read: unexpected end of file reading block header\n",
+ global->filename);
+ return &state->status;
+ }
+ if (n == -1) {
+ perror (global->filename);
+ return &state->status;
+ }
+ position += n;
+
+ r = lzma_block_header_decode (&block, NULL, header);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "%s: invalid block header (error %d)\n",
+ global->filename, r);
+ return &state->status;
+ }
+
+ /* What this actually does is it checks that the block header
+ * matches the index.
+ */
+ r = lzma_block_compressed_size (&block, iter.block.unpadded_size);
+ if (r != LZMA_OK) {
+ fprintf (stderr,
+ "%s: cannot calculate compressed size (error %d)\n",
+ global->filename, r);
+ return &state->status;
+ }
+
+ /* Where we will start writing to. */
+ oposition = iter.block.uncompressed_file_offset;
+
+ /* Read the block data and uncompress it. */
+ r = lzma_block_decoder (&strm, &block);
+ if (r != LZMA_OK) {
+ fprintf (stderr, "%s: invalid block (error %d)\n", global->filename, r);
+ return &state->status;
+ }
+
+ strm.next_in = NULL;
+ strm.avail_in = 0;
+ strm.next_out = outbuf;
+ strm.avail_out = sizeof outbuf;
+
+ for (;;) {
+ lzma_action action = LZMA_RUN;
+
+ if (strm.avail_in == 0) {
+ strm.next_in = buf;
+ n = pread (global->fd, buf, sizeof buf, position);
+ if (n == -1) {
+ perror (global->filename);
+ return &state->status;
+ }
+ position += n;
+ strm.avail_in = n;
+ if (n == 0)
+ action = LZMA_FINISH;
+ }
+
+ r = lzma_code (&strm, action);
+
+ if (strm.avail_out == 0 || r == LZMA_STREAM_END) {
+ size_t wsz = sizeof outbuf - strm.avail_out;
+
+ /* Don't write if the block is all zero, to preserve output file
+ * sparseness. However we have to update oposition.
+ */
+ if (!is_zero (outbuf, wsz)) {
+ if (pwrite (global->ofd, outbuf, wsz, oposition) != wsz) {
+ /* XXX Handle short writes. */
+ perror (global->filename);
+ return &state->status;
+ }
+ }
+ oposition += wsz;
+
+ strm.next_out = outbuf;
+ strm.avail_out = sizeof outbuf;
+ }
+
+ if (r == LZMA_STREAM_END)
+ break;
+ if (r != LZMA_OK) {
+ fprintf (stderr,
+ "%s: could not parse block data (error %d)\n",
+ global->filename, r);
+ return &state->status;
+ }
+ }
+
+ lzma_end (&strm);
+
+ for (i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i)
+ free (filters[i].options);
+ }
+
+ state->status = 0;
+ return &state->status;
+}
+
+#endif /* HAVE_LIBLZMA */
diff --git a/builder/pxzcat.ml b/builder/pxzcat.ml
new file mode 100644
index 0000000..5645370
--- /dev/null
+++ b/builder/pxzcat.ml
@@ -0,0 +1,19 @@
+(* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+external pxzcat : string -> string -> unit = "virt_builder_pxzcat"
diff --git a/builder/pxzcat.mli b/builder/pxzcat.mli
new file mode 100644
index 0000000..a2830f0
--- /dev/null
+++ b/builder/pxzcat.mli
@@ -0,0 +1,31 @@
+(* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** {1 Parallel xzcat (or fall back to regular xzcat).}
+
+ Eventually regular xzcat will be able to work in parallel and this
+ code can go away.
+*)
+
+val pxzcat : string -> string -> unit
+ (** [pxzcat input output] uncompresses the file [input] to the file
+ [output]. The input and output must both be seekable.
+
+ If liblzma was found at compile time, this uses an internal
+ implementation of parallel xzcat. Otherwise regular xzcat is
+ used. *)
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index b9fd69c..19dc3ea 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1396,10 +1396,9 @@ the output format is the same as the template format (usually raw).
=head3 pxzcat
-Virt-builder can use C<pxzcat> (parallel xzcat) if available to
-uncompress the templates. The default is to use regular C<xzcat>
-which is single-threaded. Currently this has to be compiled in,
-ie. virt-builder will probably need to be recompiled to use pxzcat.
+Virt-builder uses an internal implementation of pxzcat (parallel
+xzcat) if liblzma was found at build time. If liblzma was not found
+at build time, regular C<xzcat> is used which is single-threaded.
=head3 User-Mode Linux
diff --git a/configure.ac b/configure.ac
index 9d551af..bd3078f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -701,9 +701,15 @@ fi
dnl Check for xzcat (required).
AC_PATH_PROGS([XZCAT],[xzcat],[no])
test "x$XZCAT" = "xno" && AC_MSG_ERROR([xzcat must be installed])
+AC_DEFINE_UNQUOTED([XZCAT],["$XZCAT"],[Name of xzcat program.])
-dnl Check for pxzcat (optional).
-AC_PATH_PROGS([PXZCAT],[pxzcat],[no])
+dnl liblzma can be used by virt-builder (optional).
+PKG_CHECK_MODULES([LIBLZMA], [liblzma],[
+ AC_SUBST([LIBLZMA_CFLAGS])
+ AC_SUBST([LIBLZMA_LIBS])
+ AC_DEFINE([HAVE_LIBLZMA],[1],[liblzma found at compile time.])
+],
+[AC_MSG_WARN([liblzma not found, virt-builder will be slower])])
dnl (f)lex and bison are required for virt-builder.
AC_PROG_LEX
diff --git a/mllib/config.ml.in b/mllib/config.ml.in
index 5586c5c..d18bf7b 100644
--- a/mllib/config.ml.in
+++ b/mllib/config.ml.in
@@ -18,9 +18,3 @@
let package_name = "@PACKAGE_NAME@"
let package_version = "@PACKAGE_VERSION@"
-
-let xzcat = "@XZCAT@"
-let pxzcat =
- match "@PXZCAT@" with
- | "no" -> None
- | path -> Some path
diff --git a/po/POTFILES b/po/POTFILES
index aa52b75..4bf805f 100644
--- a/po/POTFILES
+++ b/po/POTFILES
@@ -4,6 +4,7 @@ builder/index-parser-c.c
builder/index-scan.c
builder/index-struct.c
builder/index-validate.c
+builder/pxzcat-c.c
cat/cat.c
cat/filesystems.c
cat/ls.c
diff --git a/po/POTFILES-ml b/po/POTFILES-ml
index 2ca1042..efeb75d 100644
--- a/po/POTFILES-ml
+++ b/po/POTFILES-ml
@@ -5,6 +5,7 @@ builder/get_kernel.ml
builder/index_parser.ml
builder/list_entries.ml
builder/perl_edit.ml
+builder/pxzcat.ml
builder/sigchecker.ml
mllib/common_gettext.ml
mllib/common_utils.ml
--
1.8.3.1