aredn/patches/010-lz77-decompression-support.patch

diff --git a/target/linux/generic/files/drivers/platform/mikrotik/Makefile b/target/linux/generic/files/drivers/platform/mikrotik/Makefile
index a232e1a9e8488..2e1ab3dcf44c0 100644
--- a/target/linux/generic/files/drivers/platform/mikrotik/Makefile
+++ b/target/linux/generic/files/drivers/platform/mikrotik/Makefile
@@ -1,4 +1,4 @@
 #
 # Makefile for MikroTik RouterBoard platform specific drivers
 #
-obj-$(CONFIG_MIKROTIK_RB_SYSFS)     += routerboot.o rb_hardconfig.o rb_softconfig.o
+obj-$(CONFIG_MIKROTIK_RB_SYSFS)     += routerboot.o rb_hardconfig.o rb_softconfig.o rb_hardconfig_lz77.o
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c
index bd0469d5e8385..83ba74341c8f9 100644
--- a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c
+++ b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c
@@ -38,8 +38,9 @@
 #include <linux/lzo.h>
 
 #include "routerboot.h"
+#include "rb_hardconfig_lz77.h"
 
-#define RB_HARDCONFIG_VER		"0.07"
+#define RB_HARDCONFIG_VER		"0.08"
 #define RB_HC_PR_PFX			"[rb_hardconfig] "
 
 /* ID values for hardware settings */
@@ -563,6 +564,63 @@ static int hc_wlan_data_unpack_lzor(const u16 tag_id, const u8 *inbuf, size_t in
 	return ret;
 }
 
+static int hc_wlan_data_unpack_lz77(const u16 tag_id, const u8 *inbuf, size_t inlen,
+				    void *outbuf, size_t *outlen)
+{
+	u16 rle_ofs, rle_len;
+	u8 *tempbuf;
+	size_t templen;
+	int ret;
+
+	/* Temporary buffer same size as the outbuf */
+	tempbuf = kmalloc(*outlen, GFP_KERNEL);
+	if (!tempbuf)
+		return -ENOMEM;
+
+	/* LZO-decompress lzo_len bytes of outbuf into the tempbuf */
+	ret = lz77_mikrotik_wlan_decompress(
+			(const unsigned char *)inbuf,
+			inlen,
+			(unsigned char *)tempbuf,
+			outlen);
+	if (ret < 0) {
+		pr_err(RB_HC_PR_PFX "LZ77: LZ77 decompress fail\n");
+		goto lz77_fail;
+	}
+	templen = ret;
+
+	pr_debug(RB_HC_PR_PFX "LZ77: decompressed from %zu to %d\n",
+			inlen, templen);
+
+	/* skip DRE magic */
+	tempbuf += 4;
+	templen -= 4;
+
+	/* Past magic. Look for tag node */
+	ret = routerboot_tag_find(tempbuf, templen, tag_id, &rle_ofs, &rle_len);
+	if (ret) {
+		pr_debug(RB_HC_PR_PFX "LZ77: no RLE data for id 0x%04x\n", tag_id);
+		goto lz77_fail;
+	}
+	pr_debug(RB_HC_PR_PFX "LZ77: found RLE data for id 0x%04x\n", tag_id);
+
+	if (rle_len > templen) {
+		pr_err(RB_HC_PR_PFX "LZ77: Invalid RLE data length\n");
+		ret = -EINVAL;
+		goto lz77_fail;
+	}
+
+	/* RLE-decode tempbuf back into the outbuf */
+	ret = routerboot_rle_decode(tempbuf+rle_ofs, rle_len, outbuf, outlen);
+	if (ret)
+		pr_debug(RB_HC_PR_PFX "LZ77: RLE decoding error (%d)\n", ret);
+
+lz77_fail:
+	kfree(tempbuf);
+	return ret;
+}
+
+
 static int hc_wlan_data_unpack(const u16 tag_id, const size_t tofs, size_t tlen,
 			       void *outbuf, size_t *outlen)
 {
@@ -585,6 +643,13 @@ static int hc_wlan_data_unpack(const u16 tag_id, const size_t tofs, size_t tlen,
 		tlen -= sizeof(magic);
 		ret = hc_wlan_data_unpack_lzor(tag_id, lbuf, tlen, outbuf, outlen);
 		break;
+	case RB_MAGIC_LZ77:
+		/* Skip magic */
+		lbuf += sizeof(magic);
+		tlen -= sizeof(magic);
+		ret = hc_wlan_data_unpack_lz77(tag_id, lbuf, tlen, outbuf, outlen);
+		break;
+
 	case RB_MAGIC_ERD:
 		/* Skip magic */
 		lbuf += sizeof(magic);
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.c b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.c
new file mode 100644
index 0000000000000..8e8b7a1689c87
--- /dev/null
+++ b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.c
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2023 John Thomson
+ */
+
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+
+#include "rb_hardconfig_lz77.h"
+
+#define RB_HC_PR_PFX_LZ77 "[rb_hardconfig][lz77] "
+
+
+/**
+ * lz77_mikrotik_wlan_get_bit
+ *
+ * @in:			compressed data
+ * @in_offset_bit:	bit offset to extract
+ */
+static unsigned int lz77_mikrotik_wlan_get_bit(
+		const unsigned char *in,
+		unsigned int in_offset_bit)
+{
+	return ((in[in_offset_bit>>3] >> (in_offset_bit & 7)) & 0x1);
+
+}
+
+
+/**
+ * lz77_mikrotik_wlan_get_byte
+ *
+ * @in:			compressed data
+ * @in_offset_bit:	bit offset to extract byte
+ */
+static unsigned char lz77_mikrotik_wlan_get_byte(
+		const unsigned char *in,
+		unsigned int in_offset_bit)
+{
+	unsigned char buf = 0;
+
+	/* built a byte from unaligned bits (reversed) */
+	int i; for (i = 0; i <= 7; ++i)
+		buf += ((in[(in_offset_bit+i)>>3] >> ((in_offset_bit+i) & 7)) & 0x1)<<(7-i);
+	return buf;
+}
+
+
+/**
+ * lz77_mikrotik_wlan_decode_count - decode bits at given offset as a count
+ *
+ * @in:			compressed data
+ * @in_offset_bit:	bit offset where count starts
+ * @shift:		left shift operand value of first count bit
+ * @count:		initial count
+ * @bits_used:		how many bits were consumed by this count
+ * @size:		maximum bit count for this counter
+ *
+ * Returns the decoded count
+ */
+static int lz77_mikrotik_wlan_decode_count(
+		const unsigned char *in,
+		unsigned int in_offset_bit,
+		unsigned int shift,
+		unsigned int count,
+		unsigned int *bits_used,
+		unsigned int size)
+{
+	unsigned int pos = in_offset_bit;
+	bool up = true;
+
+	size += pos;
+	*bits_used = 0;
+	pr_debug(RB_HC_PR_PFX_LZ77 "decode_count inbit: %i, start shift:%i, initial count:%i\n",
+			in_offset_bit, shift, count);
+
+	/* maybe could use find_first_zero_bit to skip count up,
+	 * and extract for the count down,
+	 * but would need to mask the bits from and to the byte align boundary
+	 */
+
+	while (true) {
+		if (pos >= size) {
+			pr_err(RB_HC_PR_PFX_LZ77 "max bit index reached before count completed\n");
+			return -1;
+		}
+
+		/* if the bit value at offset is set */
+		if (lz77_mikrotik_wlan_get_bit(in, pos)) {
+			count += (1 << shift);
+
+		/* shift increases until we find an unsed bit */
+		} else if (up) {
+			up = false;
+		}
+
+		if (up) {
+			++shift;
+		} else {
+			if (!shift) {
+				*bits_used = pos - in_offset_bit + 1;
+				return count;
+			}
+			--shift;
+		}
+
+		++pos;
+	}
+
+	return -1;
+}
+
+enum lz77_mikrotik_instruction {
+	INSTR_ERROR = -1,
+	INSTR_LITERAL_BYTE = 0,
+	/* a (non aligned) byte follows this instruction,
+	 * which is directly copied into output
+	 */
+	INSTR_PREVIOUS_OFFSET = 1,
+	/* this group is a match, with a bytes length defined by
+	 * following counter bits, starting at bitshift 0,
+	 * less the built-in count of 1
+	 * using the previous offset as source
+	 */
+	INSTR_LONG = 2
+	/* this group has two counters,
+	 * the first counter starts at bitshift 4,
+	 * 	if this counter == 0, this is a non-matching group
+	 * 	the second counter (bytes length) starts at bitshift 4,
+	 * 	less the built-in count of 11+1.
+	 * 	The final match group has this count 0,
+	 * 	and (at least 1) 0 bits pad to byte-align
+	 *
+	 * 	if this counter > 0, this is a matching group
+	 * 	this first count is the match offset (in bytes)
+	 * 	the second count is the match length (in bytes),
+	 * 	less the built-in count of 2
+	 * 	these groups can source bytes that are part of this group
+	 */
+};
+
+/**
+ * lz77_mikrotik_wlan_decode_instruction
+ *
+ * @in:			compressed data
+ * @in_offset_bit:	bit offset where instruction starts
+ * @bits_used:		how many bits were consumed by this count
+ *
+ * Returns the decoded instruction
+ */
+static enum lz77_mikrotik_instruction lz77_mikrotik_wlan_decode_instruction(
+		const unsigned char *in,
+		unsigned int in_offset_bit,
+		unsigned int *bits_used)
+{
+	if (lz77_mikrotik_wlan_get_bit(in, in_offset_bit)) {
+		*bits_used = 2;
+		if (lz77_mikrotik_wlan_get_bit(in, ++in_offset_bit)) {
+			return INSTR_LONG;
+		} else {
+			return INSTR_PREVIOUS_OFFSET;
+		}
+	} else {
+		*bits_used = 1;
+		return INSTR_LITERAL_BYTE;
+	}
+	return INSTR_ERROR;
+}
+
+struct lz77_mk_instr_opcodes {
+	/* group instruction */
+	enum lz77_mikrotik_instruction instruction;
+	/* if >0, a match group,
+	 * which starts at byte output_position - 1*offset
+	 */
+	unsigned int offset;
+	/* how long the match group is,
+	 * or how long the (following counter) non-match group is
+	 */
+	unsigned int length;
+	/* how many bits were used for this instruction + op code(s) */
+	unsigned int bits_used;
+	/* input char */
+	unsigned char *in;
+	/* offset where this instruction started */
+	unsigned int in_pos;
+};
+
+/**
+ * lz77_mikrotik_wlan_decode_instruction_operators
+ *
+ * @in:			compressed data
+ * @in_offset_bit:	bit offset where instruction starts
+ * @previous_offset:	last used match offset
+ * @opcode:		struct to hold instruction & operators
+ *
+ * Returns error code
+ */
+
+
+static int lz77_mikrotik_wlan_decode_instruction_operators(
+		const unsigned char *in,
+		unsigned int in_offset_bit,
+		unsigned int previous_offset,
+		struct lz77_mk_instr_opcodes *opcode)
+{
+	enum lz77_mikrotik_instruction instruction;
+	unsigned int bit_count = 0;
+	unsigned int bits_used = 0;
+	int offset = 0;
+	int length = 0;
+
+	instruction = lz77_mikrotik_wlan_decode_instruction(
+			in, in_offset_bit, &bit_count);
+
+	/* skip bits used by instruction */
+	bits_used += bit_count;
+
+	switch (instruction) {
+	case INSTR_LITERAL_BYTE:
+		/* non-matching char */
+		offset = 0;
+		length = 1;
+		break;
+
+	case INSTR_PREVIOUS_OFFSET:
+		/* matching group uses previous offset */
+		offset = previous_offset;
+
+		length = lz77_mikrotik_wlan_decode_count(
+				in,
+				in_offset_bit + bits_used,
+				0, 1, &bit_count,
+				LZ77_MK_MAX_COUNT_BIT_LEN);
+		if (length < 0)
+			return -1;
+		/* skip bits used by count */
+		bits_used += bit_count;
+		break;
+
+	case INSTR_LONG:
+		offset = lz77_mikrotik_wlan_decode_count(
+				in,
+				in_offset_bit + bits_used,
+				4, 0, &bit_count,
+				LZ77_MK_MAX_COUNT_BIT_LEN);
+		if (offset < 0)
+			return -1;
+
+		/* skip bits used by offset count */
+		bits_used += bit_count;
+
+		if (offset == 0) {
+			/* non-matching long group */
+			length = lz77_mikrotik_wlan_decode_count(
+					in,
+					in_offset_bit + bits_used,
+					4, 12, &bit_count,
+					LZ77_MK_MAX_COUNT_BIT_LEN);
+			if (length < 0)
+				return -1;
+			/* skip bits used by length count */
+			bits_used += bit_count;
+		} else {
+			/* matching group */
+			length = lz77_mikrotik_wlan_decode_count(
+					in,
+					in_offset_bit + bits_used,
+					0, 2, &bit_count,
+					LZ77_MK_MAX_COUNT_BIT_LEN);
+			if (length < 0)
+				return -1;
+			/* skip bits used by length count */
+			bits_used += bit_count;
+		}
+
+		break;
+
+	case INSTR_ERROR:
+		return -1;
+	}
+
+	opcode->instruction = instruction;
+	opcode->offset = offset;
+	opcode->length = length;
+	opcode->bits_used = bits_used;
+	opcode->in = (unsigned char *) in;
+	opcode->in_pos = in_offset_bit;
+	return 0;
+}
+
+/**
+ * lz77_mikrotik_wlan_decompress
+ *
+ * @in:			compressed data ptr
+ * @in_len:		length of compressed data
+ * @out:		buffer ptr to decompress into
+ * @out_len:		length of decompressed buffer
+ *
+ * Returns length of decompressed data
+ */
+int lz77_mikrotik_wlan_decompress(
+		const unsigned char *in,
+		size_t in_len,
+		unsigned char *out,
+		size_t *out_len)
+{
+	unsigned char *output_ptr;
+	unsigned int input_bit = 0;
+	unsigned char * const output_end = out + *out_len;
+	struct lz77_mk_instr_opcodes *opcode;
+	unsigned int match_offset = 0;
+	int rc = 0;
+	unsigned int match_length, partial_count;
+
+	output_ptr = out;
+
+	if (in_len > LZ77_MK_MAX_ENCODED ||
+			(in_len * 8) > UINT_MAX) {
+		pr_err(RB_HC_PR_PFX_LZ77 "input longer than expected\n");
+		return -1;
+	}
+
+	opcode = kmalloc(sizeof(struct lz77_mk_instr_opcodes), GFP_KERNEL);
+	if (!opcode)
+		return -ENOMEM;
+
+	while (true) {
+		if (output_ptr > output_end) {
+			pr_err(RB_HC_PR_PFX_LZ77 "output overrun\n");
+			goto free_lz77_struct;
+		}
+		if (input_bit > in_len*8) {
+			pr_err(RB_HC_PR_PFX_LZ77 "input overrun\n");
+			goto free_lz77_struct;
+		}
+
+		rc = lz77_mikrotik_wlan_decode_instruction_operators(
+				in, input_bit, match_offset, opcode);
+		if (rc < 0) {
+			pr_err(RB_HC_PR_PFX_LZ77 "instruction operands decode error\n");
+			goto free_lz77_struct;
+		}
+
+		pr_debug(RB_HC_PR_PFX_LZ77 "inbit:0x%x->outbyte:0x%x",
+				input_bit, output_ptr - out);
+
+		input_bit += opcode->bits_used;
+		switch (opcode->instruction) {
+		case INSTR_LITERAL_BYTE:
+			pr_debug(" short");
+			fallthrough;
+		case INSTR_LONG:
+			if (opcode->offset == 0) {
+				unsigned int i;
+				/* this is a non-matching group */
+				pr_debug(" non-match, len: 0x%x\n",
+						opcode->length);
+				/* test end marker */
+				if (opcode->length == 0xc &&
+						((input_bit + opcode->length*8) > in_len))
+				{
+
+					if (!(*(unsigned int *)out == LZ77_MK_EXPECTED_OUT)) {
+						pr_debug(RB_HC_PR_PFX_LZ77 "lz77 decompressed from %zu to %d\n",
+								in_len, output_ptr - out);
+						return (unsigned int)(output_ptr - out);
+					} else {
+						pr_err(RB_HC_PR_PFX_LZ77 "lz77 decompressed: unexpected output\n");
+						return -1;
+					}
+				}
+
+				for (i = opcode->length; i > 0; --i) {
+					*output_ptr = lz77_mikrotik_wlan_get_byte(in, input_bit);
+					++output_ptr;
+					input_bit += 8;
+				}
+				break;
+			}
+			match_offset = opcode->offset;
+			fallthrough;
+		case INSTR_PREVIOUS_OFFSET:
+			match_length = opcode->length;
+			partial_count = 0;
+
+			pr_debug(" match, offset: 0x%x, len: 0x%x",
+					opcode->offset, match_length);
+
+			if (opcode->offset == 0) {
+				pr_err(RB_HC_PR_PFX_LZ77 "match group missing opcode->offset\n");
+				goto free_lz77_struct;
+			}
+
+			/* overflow */
+			if ((output_ptr + match_length) > output_end) {
+				pr_err(RB_HC_PR_PFX_LZ77 "match group output overflow\n");
+				goto free_lz77_struct;
+			}
+
+			/* underflow */
+			if ((output_ptr - opcode->offset) < out) {
+				pr_err(RB_HC_PR_PFX_LZ77 "match group offset underflow\n");
+				goto free_lz77_struct;
+			}
+
+			while (opcode->offset < match_length) {
+				++partial_count;
+				memcpy(output_ptr,
+						output_ptr - opcode->offset,
+						opcode->offset);
+				output_ptr += opcode->offset;
+				match_length -= opcode->offset;
+			}
+			memcpy(output_ptr,
+					output_ptr - opcode->offset,
+					match_length);
+			output_ptr += match_length;
+			if (partial_count) {
+				pr_debug(" (%d partial memcpy)\n", partial_count);
+			} else {
+				pr_debug("\n");
+			}
+
+			break;
+
+		case INSTR_ERROR:
+			return -1;
+		}
+	}
+
+	pr_err(RB_HC_PR_PFX_LZ77 "decode loop broken\n");
+
+free_lz77_struct:
+	kfree(opcode);
+
+	return -1;
+}
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.h b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.h
new file mode 100644
index 0000000000000..50d19f54b3618
--- /dev/null
+++ b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2023 John Thomson
+ */
+
+#include <linux/errno.h>
+
+#define LZ77_MK_MAX_ENCODED 0x1000
+#define LZ77_MK_MAX_DECODED 0x40000
+
+/* examples have all decompressed to start with DRE\x00 */
+#define LZ77_MK_EXPECTED_OUT 0x44524500
+
+/* the look behind window
+ * unknown, for long instruction match offsets up to
+ * 6449 have been seen (would need 21 counter bits: 4 to 12 + 11 to 0)
+ * conservative value here: 27 provides offset up to 0x8000 bytes
+ */
+#define LZ77_MK_MAX_COUNT_BIT_LEN 27
+
+int lz77_mikrotik_wlan_decompress(
+		const unsigned char *in,
+		size_t in_len,
+		unsigned char *out,
+		size_t *out_len);
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h b/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h
index e858a524af43f..91693d8985226 100644
--- a/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h
+++ b/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h
@@ -16,6 +16,7 @@
 #define RB_MAGIC_SOFT	(('S') | ('o' << 8) | ('f' << 16) | ('t' << 24))
 #define RB_MAGIC_LZOR	(('L') | ('Z' << 8) | ('O' << 16) | ('R' << 24))
 #define RB_MAGIC_ERD	(('E' << 16) | ('R' << 8) | ('D'))
+#define RB_MAGIC_LZ77	(('7') | ('7' << 8) | ('Z' << 16) | ('L' << 24))
 
 #define RB_ART_SIZE	0x10000