common: move all hlq code into qso-pack.c

The split between hlq and qso-pack makes sense at a high level but results
in awkward division of code.  The simplest solution is to combine the two.
The code can be separated again if/when we support more than just HLQ file
formats.

Signed-off-by: Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
3 files changed, 637 insertions(+), 667 deletions(-)

M common/CMakeLists.txt
R common/hlq.c => 
M common/qso-pack.c
M common/CMakeLists.txt +0 -1
@@ 35,7 35,6 @@ add_library(hlogcommon SHARED
 	bands.c
 	country.c
 	csv.c
-	hlq.c
 	index.c
 	lcd.c
 	maidenhead.c

          
R common/hlq.c =>  +0 -666
@@ 1,666 0,0 @@ 
-/*
- * Copyright (c) 2020-2023 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <jeffpc/buffer.h>
-#include <jeffpc/hexdump.h>
-#include <jeffpc/base64.h>
-#include <jeffpc/rand.h>
-#include <jeffpc/time.h>
-
-#include <hlog/time.h>
-#include <hlog/qso-pack.h>
-
-#include "hlq.h"
-#include "qso-impl.h"
-
-#define BASE64_LINE_LEN 76 /* when to line-wrap base64 content */
-
-enum htype {
-	HT_INVAL = -1,
-	HT_NIL,
-	HT_INT,
-	HT_BOOL, /* added in v1 */
-	HT_TIME,
-	HT_STR_OL, /* single line string */
-	HT_STR_ML, /* multiline string */
-	HT_BLOB,
-};
-
-static const char *get_ns_name(enum qso_ns ns)
-{
-	switch (ns) {
-		case QSO_NS_DEBUG:
-			return "debug";
-		case QSO_NS_QSO:
-			return "qso";
-		case QSO_NS_TX:
-			return "tx";
-		case QSO_NS_RX:
-			return "rx";
-		case QSO_NS_ADDL:
-			return "addl";
-	}
-
-	panic("unknown hlq namespace %d", ns);
-}
-
-static int parse_ns_name(const char *str, size_t len, enum qso_ns *ns_r)
-{
-	if ((len == 5) && !memcmp(str, "debug", len))
-		*ns_r = QSO_NS_DEBUG;
-	else if ((len == 3) && !memcmp(str, "qso", len))
-		*ns_r = QSO_NS_QSO;
-	else if ((len == 2) && !memcmp(str, "tx", len))
-		*ns_r = QSO_NS_TX;
-	else if ((len == 2) && !memcmp(str, "rx", len))
-		*ns_r = QSO_NS_RX;
-	else if ((len == 4) && !memcmp(str, "addl", len))
-		*ns_r = QSO_NS_ADDL;
-	else
-		return -EINVAL; /* unknown namespace */
-
-	return 0;
-}
-
-int hlq_append_header(struct buffer *buf, const struct xuuid *uuid)
-{
-	const int ver = HLQ_VERSION_CUR;
-	const uint64_t now = gettime();
-	char id[XUUID_PRINTABLE_STRING_LENGTH];
-	char rest[32];
-	int ret;
-
-	snprintf(rest, sizeof(rest), "HLQ%c ", ver + '0');
-	ret = buffer_append_cstr(buf, rest);
-	if (ret)
-		return ret;
-
-	xuuid_unparse(uuid, id);
-	ret = buffer_append_cstr(buf, id);
-	if (ret)
-		return ret;
-
-	/* append timestamp only if writing out v0 */
-	if (ver >= 1) {
-		return buffer_append_c(buf, '\n');
-	} else {
-		snprintf(rest, sizeof(rest), " %"PRIu64"\n", now);
-
-		return buffer_append_cstr(buf, rest);
-	}
-}
-
-static int __append_key(struct buffer *buf, enum qso_ns ns, const char *key)
-{
-	int ret;
-
-	ret = buffer_append_cstr(buf, get_ns_name(ns));
-	if (ret)
-		return ret;
-
-	ret = buffer_append_c(buf, '.');
-	if (ret)
-		return ret;
-
-	return buffer_append_cstr(buf, key);
-}
-
-int hlq_append_kv_null(struct buffer *buf, enum qso_ns ns, const char *key)
-{
-	int ret;
-
-	ret = __append_key(buf, ns, key);
-	if (ret)
-		return ret;
-
-	return buffer_append_cstr(buf, " nil\n");
-}
-
-int hlq_append_kv_int(struct buffer *buf, enum qso_ns ns, const char *key,
-		      uint64_t value)
-{
-	char strval[32];
-	int ret;
-
-	snprintf(strval, sizeof(strval), "%"PRIu64, value);
-
-	ret = __append_key(buf, ns, key);
-	if (ret)
-		return ret;
-
-	ret = buffer_append_cstr(buf, " int ");
-	if (ret)
-		return ret;
-
-	ret = buffer_append_cstr(buf, strval);
-	if (ret)
-		return ret;
-
-	return buffer_append_c(buf, '\n');
-}
-
-int hlq_append_kv_bool(struct buffer *buf, enum qso_ns ns, const char *key,
-		       bool value)
-{
-	int ret;
-
-	ret = __append_key(buf, ns, key);
-	if (ret)
-		return ret;
-
-	return buffer_append_cstr(buf,
-				  value ? " bool true\n" : " bool false\n");
-}
-
-int hlq_append_kv_cstr(struct buffer *buf, enum qso_ns ns, const char *key,
-		       const char *value)
-{
-#define SENTINEL_SIZE	8
-	bool multiline = !!strchr(value, '\n');
-	char sentinel[SENTINEL_SIZE * 2 + 1];
-	int ret;
-
-	ret = __append_key(buf, ns, key);
-	if (ret)
-		return ret;
-
-	ret = buffer_append_cstr(buf, " {");
-	if (ret)
-		return ret;
-
-	if (!multiline) {
-		/* can use single-line */
-		ret = buffer_append_cstr(buf, "} ");
-		if (ret)
-			return ret;
-	} else {
-		/* must use multi-line */
-
-		/* generate a random sentinel */
-		for (;;) {
-			char tmp[SENTINEL_SIZE];
-
-			/* random hex string */
-			rand_buf(tmp, SENTINEL_SIZE);
-			hexdumpz(sentinel, tmp, SENTINEL_SIZE, false);
-
-			if (!strstr(value, sentinel))
-				break; /* not contained */
-		}
-
-		ret = buffer_append_cstr(buf, sentinel);
-		if (ret)
-			return ret;
-
-		ret = buffer_append_cstr(buf, "}\n");
-		if (ret)
-			return ret;
-	}
-
-	ret = buffer_append_cstr(buf, value);
-	if (ret)
-		return ret;
-
-	ret = buffer_append_c(buf, '\n');
-	if (ret)
-		return ret;
-
-	if (multiline) {
-		ret = buffer_append_c(buf, '{');
-		if (ret)
-			return ret;
-
-		ret = buffer_append_cstr(buf, sentinel);
-		if (ret)
-			return ret;
-
-		ret = buffer_append_cstr(buf, "}\n");
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-int hlq_append_kv_time(struct buffer *buf, enum qso_ns ns, const char *key,
-		       const struct time *time)
-{
-	char strval[32];
-	int ret;
-
-	/* need at least date or h:m time */
-	if (!(time->have & (TIME_PART_DATE | TIME_PART_TIME_HM)))
-		return hlq_append_kv_null(buf, ns, key);
-
-	ret = __append_key(buf, ns, key);
-	if (ret)
-		return ret;
-
-	ret = buffer_append_cstr(buf, " time ");
-	if (ret)
-		return ret;
-
-	time_snprint_iso8601(strval, sizeof(strval), time);
-	ret = buffer_append_cstr(buf, strval);
-	if (ret)
-		return ret;
-
-	return buffer_append_c(buf, '\n');
-}
-
-int hlq_append_kv_blob(struct buffer *buf, enum qso_ns ns, const char *key,
-		       const void *value, size_t length)
-{
-	char *out;
-	char *tmp;
-	int ret;
-
-	out = malloc(base64_required_length(length));
-	if (!out)
-		return -ENOMEM;
-
-	base64_encode(out, value, length);
-
-	ret = __append_key(buf, ns, key);
-	if (ret)
-		return ret;
-
-	ret = buffer_append_cstr(buf, " blob\n");
-	if (ret)
-		return ret;
-
-	tmp = out;
-	length = base64_required_length(length);
-
-	/* write out full lines */
-	while (length >= BASE64_LINE_LEN) {
-		ret = buffer_append(buf, tmp, BASE64_LINE_LEN);
-		if (ret)
-			return ret;
-
-		ret = buffer_append_c(buf, '\n');
-		if (ret)
-			return ret;
-
-		tmp += BASE64_LINE_LEN;
-		length -= BASE64_LINE_LEN;
-	}
-
-	/* write out partial last line, if needed */
-	if (length) {
-		ret = buffer_append(buf, tmp, length);
-		if (ret)
-			return ret;
-
-		ret = buffer_append_c(buf, '\n');
-		if (ret)
-			return ret;
-	}
-
-	return buffer_append_cstr(buf, "blobend\n");
-}
-
-int hlq_append_kv(struct buffer *buf, enum qso_ns ns, const char *key,
-		  struct val *val)
-{
-	if (!val)
-		return hlq_append_kv_null(buf, ns, key);
-
-	switch (val->type) {
-		case VT_INT:
-			return hlq_append_kv_int(buf, ns, key, val->i);
-		case VT_BOOL:
-			return hlq_append_kv_bool(buf, ns, key, val->b);
-		case VT_STR:
-			return hlq_append_kv_cstr(buf, ns, key, val_cstr(val));
-		case VT_BLOB:
-			return hlq_append_kv_blob(buf, ns, key, val->blob.ptr,
-						  val->blob.size);
-		case VT_NULL:
-		case VT_CHAR:
-		case VT_SYM:
-		case VT_ARRAY:
-		case VT_NVL:
-		case VT_CONS:
-			return -ENOTSUP;
-	}
-
-	panic("cannot set %s.%s to unknown value type %d (%s)",
-	      get_ns_name(ns), key, val->type, val_typename(val->type));
-}
-
-/* return negated errno on error, HLQ version number on success */
-int hlq_parse_header(struct buffer *buf, struct xuuid *uuid_r)
-{
-	char tmp[XUUID_PRINTABLE_STRING_LENGTH];
-	ssize_t ret;
-	int version;
-
-	/* check magic */
-	ret = buffer_read(buf, tmp, 5);
-	if (ret < 0)
-		return ret;
-	if (ret != 5)
-		return -EIO;
-
-	if (strncmp("HLQ", tmp, 3) || (tmp[4] != ' '))
-		return -EINVAL;
-
-	version = tmp[3] - '0';
-	if ((version < HLQ_VERSION_MIN) || (version > HLQ_VERSION_MAX))
-		return -EINVAL;
-
-	/* extract uuid */
-	ret = buffer_read(buf, tmp, XUUID_PRINTABLE_STRING_LENGTH);
-	if (ret < 0)
-		return ret;
-	if (ret != XUUID_PRINTABLE_STRING_LENGTH)
-		return -EIO;
-
-	if (!xuuid_parse_no_nul(uuid_r, tmp))
-		return -EINVAL;
-
-	/* check the timestamp only if reading in v0 */
-	if (version >= 1) {
-		return (tmp[36] == '\n') ? version : -EINVAL;
-	} else {
-		if (tmp[36] != ' ')
-			return -EINVAL;
-
-		/* check timestamp */
-		do {
-			ret = buffer_read(buf, tmp, 1);
-			if (ret < 0)
-				return ret;
-			if (ret != 1)
-				return -EIO;
-		} while ((*tmp >= '0') && (*tmp <= '9'));
-
-		return (*tmp == '\n') ? version : -EINVAL;
-	}
-}
-
-static enum htype parse_type_name(const char *str, size_t len)
-{
-	if ((len == 3) && !memcmp(str, "nil", len))
-		return HT_NIL;
-	else if ((len == 3) && !memcmp(str, "int", len))
-		return HT_INT;
-	else if ((len == 4) && !memcmp(str, "bool", len))
-		return HT_BOOL;
-	else if ((len == 4) && !memcmp(str, "time", len))
-		return HT_TIME;
-	else if ((len == 4) && !memcmp(str, "blob", len))
-		return HT_BLOB;
-	else if ((len == 2) && !memcmp(str, "{}", len))
-		return HT_STR_OL;
-	else if ((len > 2) && (str[0] == '{') && (str[len - 1] == '}'))
-		return HT_STR_ML;
-	else
-		return HT_INVAL;
-}
-
-static int get_line(struct buffer *buf, const char **start_r,
-		    const char **end_r)
-{
-	*start_r = buffer_data_current(buf);
-	if (!*start_r)
-		return -EIO; /* no data left */
-
-	*end_r = memchr(*start_r, '\n', buffer_remain(buf));
-	if (!*end_r)
-		return -EINVAL; /* every line must end with a '\n' */
-
-	return 0;
-}
-
-static int consume_line(struct buffer *buf, const char *start,
-			    const char *end)
-{
-	ssize_t ret;
-
-	ASSERT3P(start, <=, end);
-
-	ret = buffer_seek(buf, end - start + 1, SEEK_CUR);
-
-	return (ret < 0) ? ret : 0;
-}
-
-static struct val *parse_int(const char *str, char term)
-{
-	uint64_t tmp;
-	int ret;
-
-	ret = str2u64_full(str, &tmp, 10, term);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return VAL_ALLOC_INT(tmp);
-}
-
-static struct val *parse_bool(const char *str, const char *eol)
-{
-	size_t len = eol - str;
-
-	if ((len == 4) && !memcmp(str, "true", len))
-		return VAL_ALLOC_BOOL(true);
-	else if ((len == 5) && !memcmp(str, "false", len))
-		return VAL_ALLOC_BOOL(false);
-
-	return ERR_PTR(-EINVAL);
-}
-
-static struct val *parse_str_ol(const char *start, const char *end)
-{
-	return VAL_DUP_STR_LEN(start, end - start);
-}
-
-static struct val *parse_str_ml(struct buffer *buf, const char *type_start,
-				const char *type_end)
-{
-	struct buffer out;
-	uint8_t raw[4096]; /* FIXME */
-
-	buffer_init_static(&out, raw, 0, sizeof(raw), true);
-
-	for (;;) {
-		const char *ptr, *eol;
-		int ret;
-
-		ret = get_line(buf, &ptr, &eol);
-		if (ret)
-			return ERR_PTR(ret); /* corrupt: can't get line */
-
-		if (((eol - ptr) == (type_end - type_start - 1)) &&
-		    !memcmp(ptr, type_start, eol - ptr)) {
-			ret = consume_line(buf, ptr, eol);
-			if (ret)
-				return ERR_PTR(ret); /* seek past this line */
-			break;
-		}
-
-		ret = buffer_append(&out, ptr, eol - ptr + 1);
-		if (ret)
-			return ERR_PTR(ret);
-
-		ret = consume_line(buf, ptr, eol);
-		if (ret)
-			return ERR_PTR(ret); /* seek past this line */
-	}
-
-	if (!buffer_size(&out) ||
-	    ((buffer_size(&out) == 1) && (raw[0] == '\n')))
-		return VAL_ALLOC_NULL();
-
-	/* ignore the last line's \n */
-	return VAL_DUP_STR_LEN(buffer_data(&out), buffer_size(&out) - 1);
-}
-
-static struct val *parse_blob(struct buffer *buf)
-{
-	struct base64_decoder decoder;
-	uint8_t raw[4096]; /* FIXME */
-	ssize_t len;
-
-	base64_decode_init(&decoder, raw);
-
-	for (;;) {
-		const char *ptr, *eol;
-		int ret;
-
-		ret = get_line(buf, &ptr, &eol);
-		if (ret)
-			return ERR_PTR(ret); /* corrupt: can't get line */
-
-		if (!memcmp(ptr, "blobend\n", eol - ptr + 1)) {
-			ret = consume_line(buf, ptr, eol);
-			if (ret)
-				return ERR_PTR(ret); /* seek past this line */
-			break;
-		}
-
-		/* FIXME: hack */
-		if ((decoder.outlen + (eol - ptr)) > sizeof(raw))
-			return ERR_PTR(-ENOMEM); /* ran out of space */
-
-		if (!base64_decode_step(&decoder, ptr, eol - ptr))
-			return ERR_PTR(-EINVAL); /* corrupt: bad base64 */
-
-		ret = consume_line(buf, ptr, eol);
-		if (ret)
-			return ERR_PTR(ret); /* seek past this line */
-	}
-
-	len = base64_decode_finish(&decoder);
-	if (len < 0)
-		return ERR_PTR(-EINVAL); /* corrupt: bad base64 */
-
-	return VAL_ALLOC_BLOB_DUP(raw, len);
-}
-
-struct val *hlq_parse_kv(struct buffer *buf, int ver, enum qso_ns *ns_r,
-			 const char **key_start_r, const char **key_end_r)
-{
-	const char *ptr, *eol, *nsdot, *ktspace, *tvspace;
-	struct val *val;
-	int ret;
-
-	if (ver == HLQ_VERSION_USE_DEFAULT)
-		ver = HLQ_VERSION_CUR;
-
-	/*
-	 * Find various points of interest in the buffer
-	 */
-
-	ret = get_line(buf, &ptr, &eol);
-	if (ret)
-		return ERR_PTR(ret); /* corrupt: can't get line */
-
-	/* find namespace separator */
-	nsdot = memchr(ptr, '.', eol - ptr);
-	if (!nsdot)
-		return ERR_PTR(-EINVAL); /* corrupt: every key starts with ns */
-
-	/* find key/type space */
-	ktspace = memchr(nsdot, ' ', eol - nsdot);
-	if (!ktspace)
-		return ERR_PTR(-EINVAL); /* corrupt: every key ends with ' ' */
-
-	/*
-	 * find type/value space (except for "nil", "blob", and multiline
-	 * strings types)
-	 */
-	tvspace = memchr(ktspace + 1, ' ', eol - (ktspace + 1));
-	if (!tvspace) {
-		const char *tmp = ktspace + 1;
-		size_t len = eol - tmp;
-
-		/* type is followed by a space, unless the type is "nil" */
-		if (((len != 3) || memcmp(tmp, "nil", 3)) &&
-		    ((len != 4) || memcmp(tmp, "blob", 4)) &&
-		    ((len < 3) || (tmp[0] != '{') || (tmp[len - 1] != '}')))
-			return ERR_PTR(-EINVAL); /* corrupt: not nil/blob */
-
-		/* nil/blob/ml str, pretend that the \n is a space */
-		tvspace = eol;
-	}
-
-	/*
-	 * Parse
-	 */
-
-	/* the ns */
-	ret = parse_ns_name(ptr, nsdot - ptr, ns_r);
-	if (ret)
-		return ERR_PTR(ret);
-
-	/* the key */
-	*key_start_r = nsdot + 1;
-	*key_end_r = ktspace;
-
-	/* the type & value */
-	switch (parse_type_name(ktspace + 1, tvspace - (ktspace + 1))) {
-		case HT_INVAL:
-			return ERR_PTR(-EINVAL); /* corrupt: unknown type */
-		case HT_NIL:
-			val = NULL;
-			break;
-		case HT_INT:
-			val = parse_int(tvspace + 1, *eol);
-			break;
-		case HT_BOOL:
-			val = parse_bool(tvspace + 1, eol);
-			break;
-		case HT_TIME:
-		case HT_STR_OL:
-			val = parse_str_ol(tvspace + 1, eol);
-			break;
-		case HT_STR_ML:
-			ret = consume_line(buf, ptr, eol);
-			if (ret)
-				return ERR_PTR(ret);
-
-			/* parse_str_ml seeks over all the lines by itself */
-
-			return parse_str_ml(buf, ktspace + 1, tvspace + 1);
-		case HT_BLOB:
-			ret = consume_line(buf, ptr, eol);
-			if (ret)
-				return ERR_PTR(ret);
-
-			/* parse_blob seeks over all the lines by itself */
-
-			return parse_blob(buf);
-	}
-
-	if (IS_ERR(val))
-		return val;
-
-	/* seek the buffer since we're nearly done */
-	ret = consume_line(buf, ptr, eol);
-	if (ret) {
-		val_putref(val);
-		return ERR_PTR(ret);
-	}
-
-	return val;
-}

          
M common/qso-pack.c +637 -0
@@ 21,7 21,11 @@ 
  */
 
 #include <jeffpc/buffer.h>
+#include <jeffpc/hexdump.h>
+#include <jeffpc/base64.h>
+#include <jeffpc/rand.h>
 #include <jeffpc/nvl.h>
+#include <jeffpc/time.h>
 #include <jeffpc/error.h>
 
 #include <hlog/version.h>

          
@@ 29,6 33,8 @@ 
 #include "qso-impl.h"
 #include "hlq.h"
 
+#define BASE64_LINE_LEN 76 /* when to line-wrap base64 content */
+
 static const struct side_field {
 	const char *name;
 	struct val *(*get)(const struct qso_side *);

          
@@ 57,6 63,291 @@ static const struct side_field {
  * Packing
  */
 
+static const char *get_ns_name(enum qso_ns ns)
+{
+	switch (ns) {
+		case QSO_NS_DEBUG:
+			return "debug";
+		case QSO_NS_QSO:
+			return "qso";
+		case QSO_NS_TX:
+			return "tx";
+		case QSO_NS_RX:
+			return "rx";
+		case QSO_NS_ADDL:
+			return "addl";
+	}
+
+	panic("unknown hlq namespace %d", ns);
+}
+
+int hlq_append_header(struct buffer *buf, const struct xuuid *uuid)
+{
+	const int ver = HLQ_VERSION_CUR;
+	const uint64_t now = gettime();
+	char id[XUUID_PRINTABLE_STRING_LENGTH];
+	char rest[32];
+	int ret;
+
+	snprintf(rest, sizeof(rest), "HLQ%c ", ver + '0');
+	ret = buffer_append_cstr(buf, rest);
+	if (ret)
+		return ret;
+
+	xuuid_unparse(uuid, id);
+	ret = buffer_append_cstr(buf, id);
+	if (ret)
+		return ret;
+
+	/* append timestamp only if writing out v0 */
+	if (ver >= 1) {
+		return buffer_append_c(buf, '\n');
+	} else {
+		snprintf(rest, sizeof(rest), " %"PRIu64"\n", now);
+
+		return buffer_append_cstr(buf, rest);
+	}
+}
+
+static int __append_key(struct buffer *buf, enum qso_ns ns, const char *key)
+{
+	int ret;
+
+	ret = buffer_append_cstr(buf, get_ns_name(ns));
+	if (ret)
+		return ret;
+
+	ret = buffer_append_c(buf, '.');
+	if (ret)
+		return ret;
+
+	return buffer_append_cstr(buf, key);
+}
+
+int hlq_append_kv_null(struct buffer *buf, enum qso_ns ns, const char *key)
+{
+	int ret;
+
+	ret = __append_key(buf, ns, key);
+	if (ret)
+		return ret;
+
+	return buffer_append_cstr(buf, " nil\n");
+}
+
+int hlq_append_kv_int(struct buffer *buf, enum qso_ns ns, const char *key,
+		      uint64_t value)
+{
+	char strval[32];
+	int ret;
+
+	snprintf(strval, sizeof(strval), "%"PRIu64, value);
+
+	ret = __append_key(buf, ns, key);
+	if (ret)
+		return ret;
+
+	ret = buffer_append_cstr(buf, " int ");
+	if (ret)
+		return ret;
+
+	ret = buffer_append_cstr(buf, strval);
+	if (ret)
+		return ret;
+
+	return buffer_append_c(buf, '\n');
+}
+
+int hlq_append_kv_bool(struct buffer *buf, enum qso_ns ns, const char *key,
+		       bool value)
+{
+	int ret;
+
+	ret = __append_key(buf, ns, key);
+	if (ret)
+		return ret;
+
+	return buffer_append_cstr(buf,
+				  value ? " bool true\n" : " bool false\n");
+}
+
+int hlq_append_kv_cstr(struct buffer *buf, enum qso_ns ns, const char *key,
+		       const char *value)
+{
+#define SENTINEL_SIZE	8
+	bool multiline = !!strchr(value, '\n');
+	char sentinel[SENTINEL_SIZE * 2 + 1];
+	int ret;
+
+	ret = __append_key(buf, ns, key);
+	if (ret)
+		return ret;
+
+	ret = buffer_append_cstr(buf, " {");
+	if (ret)
+		return ret;
+
+	if (!multiline) {
+		/* can use single-line */
+		ret = buffer_append_cstr(buf, "} ");
+		if (ret)
+			return ret;
+	} else {
+		/* must use multi-line */
+
+		/* generate a random sentinel */
+		for (;;) {
+			char tmp[SENTINEL_SIZE];
+
+			/* random hex string */
+			rand_buf(tmp, SENTINEL_SIZE);
+			hexdumpz(sentinel, tmp, SENTINEL_SIZE, false);
+
+			if (!strstr(value, sentinel))
+				break; /* not contained */
+		}
+
+		ret = buffer_append_cstr(buf, sentinel);
+		if (ret)
+			return ret;
+
+		ret = buffer_append_cstr(buf, "}\n");
+		if (ret)
+			return ret;
+	}
+
+	ret = buffer_append_cstr(buf, value);
+	if (ret)
+		return ret;
+
+	ret = buffer_append_c(buf, '\n');
+	if (ret)
+		return ret;
+
+	if (multiline) {
+		ret = buffer_append_c(buf, '{');
+		if (ret)
+			return ret;
+
+		ret = buffer_append_cstr(buf, sentinel);
+		if (ret)
+			return ret;
+
+		ret = buffer_append_cstr(buf, "}\n");
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int hlq_append_kv_time(struct buffer *buf, enum qso_ns ns, const char *key,
+		       const struct time *time)
+{
+	char strval[32];
+	int ret;
+
+	/* need at least date or h:m time */
+	if (!(time->have & (TIME_PART_DATE | TIME_PART_TIME_HM)))
+		return hlq_append_kv_null(buf, ns, key);
+
+	ret = __append_key(buf, ns, key);
+	if (ret)
+		return ret;
+
+	ret = buffer_append_cstr(buf, " time ");
+	if (ret)
+		return ret;
+
+	time_snprint_iso8601(strval, sizeof(strval), time);
+	ret = buffer_append_cstr(buf, strval);
+	if (ret)
+		return ret;
+
+	return buffer_append_c(buf, '\n');
+}
+
+int hlq_append_kv_blob(struct buffer *buf, enum qso_ns ns, const char *key,
+		       const void *value, size_t length)
+{
+	char *out;
+	char *tmp;
+	int ret;
+
+	out = malloc(base64_required_length(length));
+	if (!out)
+		return -ENOMEM;
+
+	base64_encode(out, value, length);
+
+	ret = __append_key(buf, ns, key);
+	if (ret)
+		return ret;
+
+	ret = buffer_append_cstr(buf, " blob\n");
+	if (ret)
+		return ret;
+
+	tmp = out;
+	length = base64_required_length(length);
+
+	/* write out full lines */
+	while (length >= BASE64_LINE_LEN) {
+		ret = buffer_append(buf, tmp, BASE64_LINE_LEN);
+		if (ret)
+			return ret;
+
+		ret = buffer_append_c(buf, '\n');
+		if (ret)
+			return ret;
+
+		tmp += BASE64_LINE_LEN;
+		length -= BASE64_LINE_LEN;
+	}
+
+	/* write out partial last line, if needed */
+	if (length) {
+		ret = buffer_append(buf, tmp, length);
+		if (ret)
+			return ret;
+
+		ret = buffer_append_c(buf, '\n');
+		if (ret)
+			return ret;
+	}
+
+	return buffer_append_cstr(buf, "blobend\n");
+}
+
+int hlq_append_kv(struct buffer *buf, enum qso_ns ns, const char *key,
+		  struct val *val)
+{
+	if (!val)
+		return hlq_append_kv_null(buf, ns, key);
+
+	switch (val->type) {
+		case VT_INT:
+			return hlq_append_kv_int(buf, ns, key, val->i);
+		case VT_BOOL:
+			return hlq_append_kv_bool(buf, ns, key, val->b);
+		case VT_STR:
+			return hlq_append_kv_cstr(buf, ns, key, val_cstr(val));
+		case VT_BLOB:
+			return hlq_append_kv_blob(buf, ns, key, val->blob.ptr,
+						  val->blob.size);
+		case VT_NULL:
+		case VT_CHAR:
+		case VT_SYM:
+		case VT_ARRAY:
+		case VT_NVL:
+		case VT_CONS:
+			return -ENOTSUP;
+	}
+
+	panic("cannot set %s.%s to unknown value type %d (%s)",
+	      get_ns_name(ns), key, val->type, val_typename(val->type));
+}
+
 struct buffer *qso_pack_hlq(const struct qso *qso)
 {
 	const struct {

          
@@ 140,6 431,352 @@ err:
  * Unpacking
  */
 
+enum htype {
+	HT_INVAL = -1,
+	HT_NIL,
+	HT_INT,
+	HT_BOOL, /* added in v1 */
+	HT_TIME,
+	HT_STR_OL, /* single line string */
+	HT_STR_ML, /* multiline string */
+	HT_BLOB,
+};
+
+static int parse_ns_name(const char *str, size_t len, enum qso_ns *ns_r)
+{
+	if ((len == 5) && !memcmp(str, "debug", len))
+		*ns_r = QSO_NS_DEBUG;
+	else if ((len == 3) && !memcmp(str, "qso", len))
+		*ns_r = QSO_NS_QSO;
+	else if ((len == 2) && !memcmp(str, "tx", len))
+		*ns_r = QSO_NS_TX;
+	else if ((len == 2) && !memcmp(str, "rx", len))
+		*ns_r = QSO_NS_RX;
+	else if ((len == 4) && !memcmp(str, "addl", len))
+		*ns_r = QSO_NS_ADDL;
+	else
+		return -EINVAL; /* unknown namespace */
+
+	return 0;
+}
+
+/* return negated errno on error, HLQ version number on success */
+int hlq_parse_header(struct buffer *buf, struct xuuid *uuid_r)
+{
+	char tmp[XUUID_PRINTABLE_STRING_LENGTH];
+	ssize_t ret;
+	int version;
+
+	/* check magic */
+	ret = buffer_read(buf, tmp, 5);
+	if (ret < 0)
+		return ret;
+	if (ret != 5)
+		return -EIO;
+
+	if (strncmp("HLQ", tmp, 3) || (tmp[4] != ' '))
+		return -EINVAL;
+
+	version = tmp[3] - '0';
+	if ((version < HLQ_VERSION_MIN) || (version > HLQ_VERSION_MAX))
+		return -EINVAL;
+
+	/* extract uuid */
+	ret = buffer_read(buf, tmp, XUUID_PRINTABLE_STRING_LENGTH);
+	if (ret < 0)
+		return ret;
+	if (ret != XUUID_PRINTABLE_STRING_LENGTH)
+		return -EIO;
+
+	if (!xuuid_parse_no_nul(uuid_r, tmp))
+		return -EINVAL;
+
+	/* check the timestamp only if reading in v0 */
+	if (version >= 1) {
+		return (tmp[36] == '\n') ? version : -EINVAL;
+	} else {
+		if (tmp[36] != ' ')
+			return -EINVAL;
+
+		/* check timestamp */
+		do {
+			ret = buffer_read(buf, tmp, 1);
+			if (ret < 0)
+				return ret;
+			if (ret != 1)
+				return -EIO;
+		} while ((*tmp >= '0') && (*tmp <= '9'));
+
+		return (*tmp == '\n') ? version : -EINVAL;
+	}
+}
+
+static enum htype parse_type_name(const char *str, size_t len)
+{
+	if ((len == 3) && !memcmp(str, "nil", len))
+		return HT_NIL;
+	else if ((len == 3) && !memcmp(str, "int", len))
+		return HT_INT;
+	else if ((len == 4) && !memcmp(str, "bool", len))
+		return HT_BOOL;
+	else if ((len == 4) && !memcmp(str, "time", len))
+		return HT_TIME;
+	else if ((len == 4) && !memcmp(str, "blob", len))
+		return HT_BLOB;
+	else if ((len == 2) && !memcmp(str, "{}", len))
+		return HT_STR_OL;
+	else if ((len > 2) && (str[0] == '{') && (str[len - 1] == '}'))
+		return HT_STR_ML;
+	else
+		return HT_INVAL;
+}
+
+static int get_line(struct buffer *buf, const char **start_r,
+		    const char **end_r)
+{
+	*start_r = buffer_data_current(buf);
+	if (!*start_r)
+		return -EIO; /* no data left */
+
+	*end_r = memchr(*start_r, '\n', buffer_remain(buf));
+	if (!*end_r)
+		return -EINVAL; /* every line must end with a '\n' */
+
+	return 0;
+}
+
+static int consume_line(struct buffer *buf, const char *start,
+			    const char *end)
+{
+	ssize_t ret;
+
+	ASSERT3P(start, <=, end);
+
+	ret = buffer_seek(buf, end - start + 1, SEEK_CUR);
+
+	return (ret < 0) ? ret : 0;
+}
+
+static struct val *parse_int(const char *str, char term)
+{
+	uint64_t tmp;
+	int ret;
+
+	ret = str2u64_full(str, &tmp, 10, term);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return VAL_ALLOC_INT(tmp);
+}
+
+static struct val *parse_bool(const char *str, const char *eol)
+{
+	size_t len = eol - str;
+
+	if ((len == 4) && !memcmp(str, "true", len))
+		return VAL_ALLOC_BOOL(true);
+	else if ((len == 5) && !memcmp(str, "false", len))
+		return VAL_ALLOC_BOOL(false);
+
+	return ERR_PTR(-EINVAL);
+}
+
+static struct val *parse_str_ol(const char *start, const char *end)
+{
+	return VAL_DUP_STR_LEN(start, end - start);
+}
+
+static struct val *parse_str_ml(struct buffer *buf, const char *type_start,
+				const char *type_end)
+{
+	struct buffer out;
+	uint8_t raw[4096]; /* FIXME */
+
+	buffer_init_static(&out, raw, 0, sizeof(raw), true);
+
+	for (;;) {
+		const char *ptr, *eol;
+		int ret;
+
+		ret = get_line(buf, &ptr, &eol);
+		if (ret)
+			return ERR_PTR(ret); /* corrupt: can't get line */
+
+		if (((eol - ptr) == (type_end - type_start - 1)) &&
+		    !memcmp(ptr, type_start, eol - ptr)) {
+			ret = consume_line(buf, ptr, eol);
+			if (ret)
+				return ERR_PTR(ret); /* seek past this line */
+			break;
+		}
+
+		ret = buffer_append(&out, ptr, eol - ptr + 1);
+		if (ret)
+			return ERR_PTR(ret);
+
+		ret = consume_line(buf, ptr, eol);
+		if (ret)
+			return ERR_PTR(ret); /* seek past this line */
+	}
+
+	if (!buffer_size(&out) ||
+	    ((buffer_size(&out) == 1) && (raw[0] == '\n')))
+		return VAL_ALLOC_NULL();
+
+	/* ignore the last line's \n */
+	return VAL_DUP_STR_LEN(buffer_data(&out), buffer_size(&out) - 1);
+}
+
+static struct val *parse_blob(struct buffer *buf)
+{
+	struct base64_decoder decoder;
+	uint8_t raw[4096]; /* FIXME */
+	ssize_t len;
+
+	base64_decode_init(&decoder, raw);
+
+	for (;;) {
+		const char *ptr, *eol;
+		int ret;
+
+		ret = get_line(buf, &ptr, &eol);
+		if (ret)
+			return ERR_PTR(ret); /* corrupt: can't get line */
+
+		if (!memcmp(ptr, "blobend\n", eol - ptr + 1)) {
+			ret = consume_line(buf, ptr, eol);
+			if (ret)
+				return ERR_PTR(ret); /* seek past this line */
+			break;
+		}
+
+		/* FIXME: hack */
+		if ((decoder.outlen + (eol - ptr)) > sizeof(raw))
+			return ERR_PTR(-ENOMEM); /* ran out of space */
+
+		if (!base64_decode_step(&decoder, ptr, eol - ptr))
+			return ERR_PTR(-EINVAL); /* corrupt: bad base64 */
+
+		ret = consume_line(buf, ptr, eol);
+		if (ret)
+			return ERR_PTR(ret); /* seek past this line */
+	}
+
+	len = base64_decode_finish(&decoder);
+	if (len < 0)
+		return ERR_PTR(-EINVAL); /* corrupt: bad base64 */
+
+	return VAL_ALLOC_BLOB_DUP(raw, len);
+}
+
+struct val *hlq_parse_kv(struct buffer *buf, int ver, enum qso_ns *ns_r,
+			 const char **key_start_r, const char **key_end_r)
+{
+	const char *ptr, *eol, *nsdot, *ktspace, *tvspace;
+	struct val *val;
+	int ret;
+
+	if (ver == HLQ_VERSION_USE_DEFAULT)
+		ver = HLQ_VERSION_CUR;
+
+	/*
+	 * Find various points of interest in the buffer
+	 */
+
+	ret = get_line(buf, &ptr, &eol);
+	if (ret)
+		return ERR_PTR(ret); /* corrupt: can't get line */
+
+	/* find namespace separator */
+	nsdot = memchr(ptr, '.', eol - ptr);
+	if (!nsdot)
+		return ERR_PTR(-EINVAL); /* corrupt: every key starts with ns */
+
+	/* find key/type space */
+	ktspace = memchr(nsdot, ' ', eol - nsdot);
+	if (!ktspace)
+		return ERR_PTR(-EINVAL); /* corrupt: every key ends with ' ' */
+
+	/*
+	 * find type/value space (except for "nil", "blob", and multiline
+	 * strings types)
+	 */
+	tvspace = memchr(ktspace + 1, ' ', eol - (ktspace + 1));
+	if (!tvspace) {
+		const char *tmp = ktspace + 1;
+		size_t len = eol - tmp;
+
+		/* type is followed by a space, unless the type is "nil" */
+		if (((len != 3) || memcmp(tmp, "nil", 3)) &&
+		    ((len != 4) || memcmp(tmp, "blob", 4)) &&
+		    ((len < 3) || (tmp[0] != '{') || (tmp[len - 1] != '}')))
+			return ERR_PTR(-EINVAL); /* corrupt: not nil/blob */
+
+		/* nil/blob/ml str, pretend that the \n is a space */
+		tvspace = eol;
+	}
+
+	/*
+	 * Parse
+	 */
+
+	/* the ns */
+	ret = parse_ns_name(ptr, nsdot - ptr, ns_r);
+	if (ret)
+		return ERR_PTR(ret);
+
+	/* the key */
+	*key_start_r = nsdot + 1;
+	*key_end_r = ktspace;
+
+	/* the type & value */
+	switch (parse_type_name(ktspace + 1, tvspace - (ktspace + 1))) {
+		case HT_INVAL:
+			return ERR_PTR(-EINVAL); /* corrupt: unknown type */
+		case HT_NIL:
+			val = NULL;
+			break;
+		case HT_INT:
+			val = parse_int(tvspace + 1, *eol);
+			break;
+		case HT_BOOL:
+			val = parse_bool(tvspace + 1, eol);
+			break;
+		case HT_TIME:
+		case HT_STR_OL:
+			val = parse_str_ol(tvspace + 1, eol);
+			break;
+		case HT_STR_ML:
+			ret = consume_line(buf, ptr, eol);
+			if (ret)
+				return ERR_PTR(ret);
+
+			/* parse_str_ml seeks over all the lines by itself */
+
+			return parse_str_ml(buf, ktspace + 1, tvspace + 1);
+		case HT_BLOB:
+			ret = consume_line(buf, ptr, eol);
+			if (ret)
+				return ERR_PTR(ret);
+
+			/* parse_blob seeks over all the lines by itself */
+
+			return parse_blob(buf);
+	}
+
+	if (IS_ERR(val))
+		return val;
+
+	/* seek the buffer since we're nearly done */
+	ret = consume_line(buf, ptr, eol);
+	if (ret) {
+		val_putref(val);
+		return ERR_PTR(ret);
+	}
+
+	return val;
+}
+
 static int __unpack_qso_ns(struct qso *qso, const char *key, struct val *val)
 {
 	int ret;