Mirrors
/
aredn
mirror of https://github.com/aredn/aredn.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add LZ77 decompression for newer Mikrotik devices (#920) 

* Incorporate the lz77 decompression patch.
Problem identified and solution proposed here
https://github.com/aredn/aredn/issues/919

* Compiler syntax fixes
This commit is contained in:
Tim Wilkinson 2023-08-30 20:18:13 -07:00 committed by GitHub
parent 7ad310b036
commit 9beaa03ef7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 602 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

601
patches/010-lz77-decompression-support.patch Normal file
View File

@ -0,0 +1,601 @@
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/Makefile b/target/linux/generic/files/drivers/platform/mikrotik/Makefile
index a232e1a9e8488..2e1ab3dcf44c0 100644
--- a/target/linux/generic/files/drivers/platform/mikrotik/Makefile
+++ b/target/linux/generic/files/drivers/platform/mikrotik/Makefile
@@ -2,3 +2,6 @@
# Makefile for MikroTik RouterBoard platform specific drivers
#
obj-$(CONFIG_MIKROTIK_RB_SYSFS) += routerboot.o rb_hardconfig.o rb_softconfig.o
+ifneq ($(filter y,$(CONFIG_CPU_LITTLE_ENDIAN) $(CONFIG_ARCH_IPQ40XX)),)
+obj-y += rb_hardconfig_lz77.o
+endif
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c
index bd0469d5e8385..83ba74341c8f9 100644
--- a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c
+++ b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig.c
@@ -38,8 +38,9 @@
#include <linux/lzo.h>
#include "routerboot.h"
+#include "rb_hardconfig_lz77.h"
-#define RB_HARDCONFIG_VER "0.07"
+#define RB_HARDCONFIG_VER "0.08"
#define RB_HC_PR_PFX "[rb_hardconfig] "
/* ID values for hardware settings */
@@ -563,6 +564,63 @@ static int hc_wlan_data_unpack_lzor(const u16 tag_id, const u8 *inbuf, size_t in
return ret;
}
+static int hc_wlan_data_unpack_lz77(const u16 tag_id, const u8 *inbuf, size_t inlen,
+ void *outbuf, size_t *outlen)
+{
+ u16 rle_ofs, rle_len;
+ u8 *tempbuf;
+ size_t templen;
+ int ret;
+
+ /* Temporary buffer same size as the outbuf */
+ tempbuf = kmalloc(*outlen, GFP_KERNEL);
+ if (!tempbuf)
+ return -ENOMEM;
+
+ /* LZO-decompress lzo_len bytes of outbuf into the tempbuf */
+ ret = lz77_mikrotik_wlan_decompress(
+ (const unsigned char *)inbuf,
+ inlen,
+ (unsigned char *)tempbuf,
+ outlen);
+ if (ret < 0) {
+ pr_err(RB_HC_PR_PFX "LZ77: LZ77 decompress fail\n");
+ goto lz77_fail;
+ }
+ templen = ret;
+
+ pr_debug(RB_HC_PR_PFX "LZ77: decompressed from %zu to %d\n",
+ inlen, templen);
+
+ /* skip DRE magic */
+ tempbuf += 4;
+ templen -= 4;
+
+ /* Past magic. Look for tag node */
+ ret = routerboot_tag_find(tempbuf, templen, tag_id, &rle_ofs, &rle_len);
+ if (ret) {
+ pr_debug(RB_HC_PR_PFX "LZ77: no RLE data for id 0x%04x\n", tag_id);
+ goto lz77_fail;
+ }
+ pr_debug(RB_HC_PR_PFX "LZ77: found RLE data for id 0x%04x\n", tag_id);
+
+ if (rle_len > templen) {
+ pr_err(RB_HC_PR_PFX "LZ77: Invalid RLE data length\n");
+ ret = -EINVAL;
+ goto lz77_fail;
+ }
+
+ /* RLE-decode tempbuf back into the outbuf */
+ ret = routerboot_rle_decode(tempbuf+rle_ofs, rle_len, outbuf, outlen);
+ if (ret)
+ pr_debug(RB_HC_PR_PFX "LZ77: RLE decoding error (%d)\n", ret);
+
+lz77_fail:
+ kfree(tempbuf);
+ return ret;
+}
+
+
static int hc_wlan_data_unpack(const u16 tag_id, const size_t tofs, size_t tlen,
void *outbuf, size_t *outlen)
{
@@ -585,6 +643,13 @@ static int hc_wlan_data_unpack(const u16 tag_id, const size_t tofs, size_t tlen,
tlen -= sizeof(magic);
ret = hc_wlan_data_unpack_lzor(tag_id, lbuf, tlen, outbuf, outlen);
break;
+ case RB_MAGIC_LZ77:
+ /* Skip magic */
+ lbuf += sizeof(magic);
+ tlen -= sizeof(magic);
+ ret = hc_wlan_data_unpack_lz77(tag_id, lbuf, tlen, outbuf, outlen);
+ break;
+
case RB_MAGIC_ERD:
/* Skip magic */
lbuf += sizeof(magic);
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.c b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.c
new file mode 100644
index 0000000000000..8e8b7a1689c87
--- /dev/null
+++ b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.c
@@ -0,0 +1,437 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2023 John Thomson
+ */
+
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+
+#include "rb_hardconfig_lz77.h"
+
+#define RB_HC_PR_PFX_LZ77 "[rb_hardconfig][lz77] "
+
+
+/**
+ * lz77_mikrotik_wlan_get_bit
+ *
+ * @in: compressed data
+ * @in_offset_bit: bit offset to extract
+ */
+static unsigned int lz77_mikrotik_wlan_get_bit(
+ const unsigned char *in,
+ unsigned int in_offset_bit)
+{
+ return ((in[in_offset_bit>>3] >> (in_offset_bit & 7)) & 0x1);
+
+}
+
+
+/**
+ * lz77_mikrotik_wlan_get_byte
+ *
+ * @in: compressed data
+ * @in_offset_bit: bit offset to extract byte
+ */
+static unsigned char lz77_mikrotik_wlan_get_byte(
+ const unsigned char *in,
+ unsigned int in_offset_bit)
+{
+ unsigned char buf = 0;
+
+ /* built a byte from unaligned bits (reversed) */
+ int i; for (i = 0; i <= 7; ++i)
+ buf += ((in[(in_offset_bit+i)>>3] >> ((in_offset_bit+i) & 7)) & 0x1)<<(7-i);
+ return buf;
+}
+
+
+/**
+ * lz77_mikrotik_wlan_decode_count - decode bits at given offset as a count
+ *
+ * @in: compressed data
+ * @in_offset_bit: bit offset where count starts
+ * @shift: left shift operand value of first count bit
+ * @count: initial count
+ * @bits_used: how many bits were consumed by this count
+ * @size: maximum bit count for this counter
+ *
+ * Returns the decoded count
+ */
+static int lz77_mikrotik_wlan_decode_count(
+ const unsigned char *in,
+ unsigned int in_offset_bit,
+ unsigned int shift,
+ unsigned int count,
+ unsigned int *bits_used,
+ unsigned int size)
+{
+ unsigned int pos = in_offset_bit;
+ bool up = true;
+
+ size += pos;
+ *bits_used = 0;
+ pr_debug(RB_HC_PR_PFX_LZ77 "decode_count inbit: %i, start shift:%i, initial count:%i\n",
+ in_offset_bit, shift, count);
+
+ /* maybe could use find_first_zero_bit to skip count up,
+ * and extract for the count down,
+ * but would need to mask the bits from and to the byte align boundary
+ */
+
+ while (true) {
+ if (pos >= size) {
+ pr_err(RB_HC_PR_PFX_LZ77 "max bit index reached before count completed\n");
+ return -1;
+ }
+
+ /* if the bit value at offset is set */
+ if (lz77_mikrotik_wlan_get_bit(in, pos)) {
+ count += (1 << shift);
+
+ /* shift increases until we find an unsed bit */
+ } else if (up) {
+ up = false;
+ }
+
+ if (up) {
+ ++shift;
+ } else {
+ if (!shift) {
+ *bits_used = pos - in_offset_bit + 1;
+ return count;
+ }
+ --shift;
+ }
+
+ ++pos;
+ }
+
+ return -1;
+}
+
+enum lz77_mikrotik_instruction {
+ INSTR_ERROR = -1,
+ INSTR_LITERAL_BYTE = 0,
+ /* a (non aligned) byte follows this instruction,
+ * which is directly copied into output
+ */
+ INSTR_PREVIOUS_OFFSET = 1,
+ /* this group is a match, with a bytes length defined by
+ * following counter bits, starting at bitshift 0,
+ * less the built-in count of 1
+ * using the previous offset as source
+ */
+ INSTR_LONG = 2
+ /* this group has two counters,
+ * the first counter starts at bitshift 4,
+ * if this counter == 0, this is a non-matching group
+ * the second counter (bytes length) starts at bitshift 4,
+ * less the built-in count of 11+1.
+ * The final match group has this count 0,
+ * and (at least 1) 0 bits pad to byte-align
+ *
+ * if this counter > 0, this is a matching group
+ * this first count is the match offset (in bytes)
+ * the second count is the match length (in bytes),
+ * less the built-in count of 2
+ * these groups can source bytes that are part of this group
+ */
+};
+
+/**
+ * lz77_mikrotik_wlan_decode_instruction
+ *
+ * @in: compressed data
+ * @in_offset_bit: bit offset where instruction starts
+ * @bits_used: how many bits were consumed by this count
+ *
+ * Returns the decoded instruction
+ */
+static enum lz77_mikrotik_instruction lz77_mikrotik_wlan_decode_instruction(
+ const unsigned char *in,
+ unsigned int in_offset_bit,
+ unsigned int *bits_used)
+{
+ if (lz77_mikrotik_wlan_get_bit(in, in_offset_bit)) {
+ *bits_used = 2;
+ if (lz77_mikrotik_wlan_get_bit(in, ++in_offset_bit)) {
+ return INSTR_LONG;
+ } else {
+ return INSTR_PREVIOUS_OFFSET;
+ }
+ } else {
+ *bits_used = 1;
+ return INSTR_LITERAL_BYTE;
+ }
+ return INSTR_ERROR;
+}
+
+struct lz77_mk_instr_opcodes {
+ /* group instruction */
+ enum lz77_mikrotik_instruction instruction;
+ /* if >0, a match group,
+ * which starts at byte output_position - 1*offset
+ */
+ unsigned int offset;
+ /* how long the match group is,
+ * or how long the (following counter) non-match group is
+ */
+ unsigned int length;
+ /* how many bits were used for this instruction + op code(s) */
+ unsigned int bits_used;
+ /* input char */
+ unsigned char *in;
+ /* offset where this instruction started */
+ unsigned int in_pos;
+};
+
+/**
+ * lz77_mikrotik_wlan_decode_instruction_operators
+ *
+ * @in: compressed data
+ * @in_offset_bit: bit offset where instruction starts
+ * @previous_offset: last used match offset
+ * @opcode: struct to hold instruction & operators
+ *
+ * Returns error code
+ */
+
+
+static int lz77_mikrotik_wlan_decode_instruction_operators(
+ const unsigned char *in,
+ unsigned int in_offset_bit,
+ unsigned int previous_offset,
+ struct lz77_mk_instr_opcodes *opcode)
+{
+ enum lz77_mikrotik_instruction instruction;
+ unsigned int bit_count = 0;
+ unsigned int bits_used = 0;
+ int offset = 0;
+ int length = 0;
+
+ instruction = lz77_mikrotik_wlan_decode_instruction(
+ in, in_offset_bit, &bit_count);
+
+ /* skip bits used by instruction */
+ bits_used += bit_count;
+
+ switch (instruction) {
+ case INSTR_LITERAL_BYTE:
+ /* non-matching char */
+ offset = 0;
+ length = 1;
+ break;
+
+ case INSTR_PREVIOUS_OFFSET:
+ /* matching group uses previous offset */
+ offset = previous_offset;
+
+ length = lz77_mikrotik_wlan_decode_count(
+ in,
+ in_offset_bit + bits_used,
+ 0, 1, &bit_count,
+ LZ77_MK_MAX_COUNT_BIT_LEN);
+ if (length < 0)
+ return -1;
+ /* skip bits used by count */
+ bits_used += bit_count;
+ break;
+
+ case INSTR_LONG:
+ offset = lz77_mikrotik_wlan_decode_count(
+ in,
+ in_offset_bit + bits_used,
+ 4, 0, &bit_count,
+ LZ77_MK_MAX_COUNT_BIT_LEN);
+ if (offset < 0)
+ return -1;
+
+ /* skip bits used by offset count */
+ bits_used += bit_count;
+
+ if (offset == 0) {
+ /* non-matching long group */
+ length = lz77_mikrotik_wlan_decode_count(
+ in,
+ in_offset_bit + bits_used,
+ 4, 12, &bit_count,
+ LZ77_MK_MAX_COUNT_BIT_LEN);
+ if (length < 0)
+ return -1;
+ /* skip bits used by length count */
+ bits_used += bit_count;
+ } else {
+ /* matching group */
+ length = lz77_mikrotik_wlan_decode_count(
+ in,
+ in_offset_bit + bits_used,
+ 0, 2, &bit_count,
+ LZ77_MK_MAX_COUNT_BIT_LEN);
+ if (length < 0)
+ return -1;
+ /* skip bits used by length count */
+ bits_used += bit_count;
+ }
+
+ break;
+
+ case INSTR_ERROR:
+ return -1;
+ }
+
+ opcode->instruction = instruction;
+ opcode->offset = offset;
+ opcode->length = length;
+ opcode->bits_used = bits_used;
+ opcode->in = (unsigned char *) in;
+ opcode->in_pos = in_offset_bit;
+ return 0;
+}
+
+/**
+ * lz77_mikrotik_wlan_decompress
+ *
+ * @in: compressed data ptr
+ * @in_len: length of compressed data
+ * @out: buffer ptr to decompress into
+ * @out_len: length of decompressed buffer
+ *
+ * Returns length of decompressed data
+ */
+int lz77_mikrotik_wlan_decompress(
+ const unsigned char *in,
+ size_t in_len,
+ unsigned char *out,
+ size_t *out_len)
+{
+ unsigned char *output_ptr;
+ unsigned int input_bit = 0;
+ unsigned char * const output_end = out + *out_len;
+ struct lz77_mk_instr_opcodes *opcode;
+ unsigned int match_offset = 0;
+ int rc = 0;
+ unsigned int match_length, partial_count;
+
+ output_ptr = out;
+
+ if (in_len > LZ77_MK_MAX_ENCODED ||
+ (in_len * 8) > UINT_MAX) {
+ pr_err(RB_HC_PR_PFX_LZ77 "input longer than expected\n");
+ return -1;
+ }
+
+ opcode = kmalloc(sizeof(struct lz77_mk_instr_opcodes), GFP_KERNEL);
+ if (!opcode)
+ return -ENOMEM;
+
+ while (true) {
+ if (output_ptr > output_end) {
+ pr_err(RB_HC_PR_PFX_LZ77 "output overrun\n");
+ goto free_lz77_struct;
+ }
+ if (input_bit > in_len*8) {
+ pr_err(RB_HC_PR_PFX_LZ77 "input overrun\n");
+ goto free_lz77_struct;
+ }
+
+ rc = lz77_mikrotik_wlan_decode_instruction_operators(
+ in, input_bit, match_offset, opcode);
+ if (rc < 0) {
+ pr_err(RB_HC_PR_PFX_LZ77 "instruction operands decode error\n");
+ goto free_lz77_struct;
+ }
+
+ pr_debug(RB_HC_PR_PFX_LZ77 "inbit:0x%x->outbyte:0x%x",
+ input_bit, output_ptr - out);
+
+ input_bit += opcode->bits_used;
+ switch (opcode->instruction) {
+ case INSTR_LITERAL_BYTE:
+ pr_debug(" short");
+ fallthrough;
+ case INSTR_LONG:
+ if (opcode->offset == 0) {
+ /* this is a non-matching group */
+ pr_debug(" non-match, len: 0x%x\n",
+ opcode->length);
+ /* test end marker */
+ if (opcode->length == 0xc &&
+ ((input_bit + opcode->length*8) > in_len))
+ {
+
+ if (!(*(unsigned int *)out == LZ77_MK_EXPECTED_OUT)) {
+ pr_debug(RB_HC_PR_PFX_LZ77 "lz77 decompressed from %zu to %d\n",
+ in_len, output_ptr - out);
+ return (unsigned int)(output_ptr - out);
+ } else {
+ pr_err(RB_HC_PR_PFX_LZ77 "lz77 decompressed: unexpected output\n");
+ return -1;
+ }
+ }
+
+ unsigned int i; for (i = opcode->length; i > 0; --i) {
+ *output_ptr = lz77_mikrotik_wlan_get_byte(in, input_bit);
+ ++output_ptr;
+ input_bit += 8;
+ }
+ break;
+ }
+ match_offset = opcode->offset;
+ fallthrough;
+ case INSTR_PREVIOUS_OFFSET:
+ match_length = opcode->length;
+ partial_count = 0;
+
+ pr_debug(" match, offset: 0x%x, len: 0x%x",
+ opcode->offset, match_length);
+
+ if (opcode->offset == 0) {
+ pr_err(RB_HC_PR_PFX_LZ77 "match group missing opcode->offset\n");
+ goto free_lz77_struct;
+ }
+
+ /* overflow */
+ if ((output_ptr + match_length) > output_end) {
+ pr_err(RB_HC_PR_PFX_LZ77 "match group output overflow\n");
+ goto free_lz77_struct;
+ }
+
+ /* underflow */
+ if ((output_ptr - opcode->offset) < out) {
+ pr_err(RB_HC_PR_PFX_LZ77 "match group offset underflow\n");
+ goto free_lz77_struct;
+ }
+
+ while (opcode->offset < match_length) {
+ ++partial_count;
+ memcpy(output_ptr,
+ output_ptr - opcode->offset,
+ opcode->offset);
+ output_ptr += opcode->offset;
+ match_length -= opcode->offset;
+ }
+ memcpy(output_ptr,
+ output_ptr - opcode->offset,
+ match_length);
+ output_ptr += match_length;
+ if (partial_count) {
+ pr_debug(" (%d partial memcpy)\n", partial_count);
+ } else {
+ pr_debug("\n");
+ }
+
+ break;
+
+ case INSTR_ERROR:
+ return -1;
+ }
+ }
+
+ pr_err(RB_HC_PR_PFX_LZ77 "decode loop broken\n");
+
+free_lz77_struct:
+ kfree(opcode);
+
+ return -1;
+}
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.h b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.h
new file mode 100644
index 0000000000000..50d19f54b3618
--- /dev/null
+++ b/target/linux/generic/files/drivers/platform/mikrotik/rb_hardconfig_lz77.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2023 John Thomson
+ */
+
+#include <linux/errno.h>
+
+#define LZ77_MK_MAX_ENCODED 0x1000
+#define LZ77_MK_MAX_DECODED 0x40000
+
+/* examples have all decompressed to start with DRE\x00 */
+#define LZ77_MK_EXPECTED_OUT 0x44524500
+
+/* the look behind window
+ * unknown, for long instruction match offsets up to
+ * 6449 have been seen (would need 21 counter bits: 4 to 12 + 11 to 0)
+ * conservative value here: 27 provides offset up to 0x8000 bytes
+ */
+#define LZ77_MK_MAX_COUNT_BIT_LEN 27
+
+#if defined(CONFIG_CPU_LITTLE_ENDIAN) || defined(CONFIG_ARCH_IPQ40XX)
+int lz77_mikrotik_wlan_decompress(
+ const unsigned char *in,
+ size_t in_len,
+ unsigned char *out,
+ size_t *out_len);
+#else
+static inline int lz77_mikrotik_wlan_decompress(
+ const unsigned char *in,
+ size_t in_len,
+ unsigned char *out,
+ size_t *out_len)
+{
+ return -EOPNOTSUPP;
+}
+#endif /* defined(CONFIG_CPU_LITTLE_ENDIAN) || defined(CONFIG_ARCH_IPQ40XX) */
diff --git a/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h b/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h
index e858a524af43f..91693d8985226 100644
--- a/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h
+++ b/target/linux/generic/files/drivers/platform/mikrotik/routerboot.h
@@ -16,6 +16,7 @@
#define RB_MAGIC_SOFT (('S') | ('o' << 8) | ('f' << 16) | ('t' << 24))
#define RB_MAGIC_LZOR (('L') | ('Z' << 8) | ('O' << 16) | ('R' << 24))
#define RB_MAGIC_ERD (('E' << 16) | ('R' << 8) | ('D'))
+#define RB_MAGIC_LZ77 (('7') | ('7' << 8) | ('Z' << 16) | ('L' << 24))
#define RB_ART_SIZE 0x10000

1
patches/series
View File

@ -2,6 +2,7 @@
001-ath79-cpe220v3-sysupgrade-supported.patch
001-ath79-reverse-wpad-basic-wolfssl.patch
006-rocket-m-flash-fix.patch
010-lz77-decompression-support.patch
100-remove-rcbutton-reset.patch
701-ath9k-reset.patch
701-extended-spectrum.patch