From 99951676c0e0004a122770d2ecf5d4dd0b9b53b2 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 14 Apr 2014 17:38:35 +0200 Subject: [PATCH] Successfull compression test with heatshrink --- Modem/Modem_user.mk | 2 + Modem/compression/fastlz.c | 543 --------------------- Modem/compression/fastlz.h | 100 ---- Modem/compression/heatshrink_common.h | 19 + Modem/compression/heatshrink_config.h | 24 + Modem/compression/heatshrink_decoder.c | 376 +++++++++++++++ Modem/compression/heatshrink_decoder.h | 101 ++++ Modem/compression/heatshrink_encoder.c | 641 +++++++++++++++++++++++++ Modem/compression/heatshrink_encoder.h | 109 +++++ Modem/compression/lzfx.c | 362 -------------- Modem/compression/lzfx.h | 90 ---- Modem/main.c | 120 ++++- Modem/protocol/mp1.h | 2 - buildrev.h | 2 +- 14 files changed, 1386 insertions(+), 1105 deletions(-) delete mode 100644 Modem/compression/fastlz.c delete mode 100644 Modem/compression/fastlz.h create mode 100644 Modem/compression/heatshrink_common.h create mode 100644 Modem/compression/heatshrink_config.h create mode 100644 Modem/compression/heatshrink_decoder.c create mode 100644 Modem/compression/heatshrink_decoder.h create mode 100644 Modem/compression/heatshrink_encoder.c create mode 100644 Modem/compression/heatshrink_encoder.h delete mode 100644 Modem/compression/lzfx.c delete mode 100644 Modem/compression/lzfx.h diff --git a/Modem/Modem_user.mk b/Modem/Modem_user.mk index 2b3a404..57a4629 100644 --- a/Modem/Modem_user.mk +++ b/Modem/Modem_user.mk @@ -13,6 +13,8 @@ Modem_USER_CSRC = \ $(Modem_HW_PATH)/hardware.c \ $(Modem_HW_PATH)/afsk.c \ $(Modem_HW_PATH)/protocol/mp1.c \ + $(Modem_HW_PATH)/compression/heatshrink_decoder.c \ + $(Modem_HW_PATH)/compression/heatshrink_encoder.c \ # # Files included by the user. diff --git a/Modem/compression/fastlz.c b/Modem/compression/fastlz.c deleted file mode 100644 index 4273d51..0000000 --- a/Modem/compression/fastlz.c +++ /dev/null @@ -1,543 +0,0 @@ -/* - FastLZ - lightning-fast lossless compression library - - Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#if !defined(FASTLZ__COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) - -/* - * Always check for bound when decompressing. - * Generally it is best to leave it defined. - */ -#define FASTLZ_SAFE - -/* - * Give hints to the compiler for branch prediction optimization. - */ -#if defined(__GNUC__) && (__GNUC__ > 2) -#define FASTLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) -#define FASTLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) -#else -#define FASTLZ_EXPECT_CONDITIONAL(c) (c) -#define FASTLZ_UNEXPECT_CONDITIONAL(c) (c) -#endif - -/* - * Use inlined functions for supported systems. - */ -#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C) -#define FASTLZ_INLINE inline -#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__) -#define FASTLZ_INLINE __inline -#else -#define FASTLZ_INLINE -#endif - -/* - * Prevent accessing more than 8-bit at once, except on x86 architectures. - */ -#if !defined(FASTLZ_STRICT_ALIGN) -#define FASTLZ_STRICT_ALIGN -#if defined(__i386__) || defined(__386) /* GNU C, Sun Studio */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(_M_IX86) /* Intel, MSVC */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__386) -#undef FASTLZ_STRICT_ALIGN -#elif defined(_X86_) /* MinGW */ -#undef FASTLZ_STRICT_ALIGN -#elif defined(__I86__) /* Digital Mars */ -#undef FASTLZ_STRICT_ALIGN -#endif -#endif - -/* - * FIXME: use preprocessor magic to set this on different platforms! - */ -typedef unsigned char flzuint8; -typedef unsigned short flzuint16; -typedef unsigned int flzuint32; - -/* prototypes */ -int fastlz_compress(const void* input, int length, void* output); -int fastlz_compress_level(int level, const void* input, int length, void* output); -int fastlz_decompress(const void* input, int length, void* output, int maxout); - -#define MAX_COPY 32 -#define MAX_LEN 264 /* 256 + 8 */ -#define MAX_DISTANCE 8192 - -#if !defined(FASTLZ_STRICT_ALIGN) -#define FASTLZ_READU16(p) *((const flzuint16*)(p)) -#else -#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8) -#endif - -#define HASH_LOG 13 -#define HASH_SIZE (1<< HASH_LOG) -#define HASH_MASK (HASH_SIZE-1) -#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; } - -#undef FASTLZ_LEVEL -#define FASTLZ_LEVEL 1 - -#undef FASTLZ_COMPRESSOR -#undef FASTLZ_DECOMPRESSOR -#define FASTLZ_COMPRESSOR fastlz1_compress -#define FASTLZ_DECOMPRESSOR fastlz1_decompress -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); -#include "fastlz.c" - -#undef FASTLZ_LEVEL -#define FASTLZ_LEVEL 2 - -#undef MAX_DISTANCE -#define MAX_DISTANCE 8191 -#define MAX_FARDISTANCE (65535+MAX_DISTANCE-1) - -#undef FASTLZ_COMPRESSOR -#undef FASTLZ_DECOMPRESSOR -#define FASTLZ_COMPRESSOR fastlz2_compress -#define FASTLZ_DECOMPRESSOR fastlz2_decompress -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); -#include "fastlz.c" - -int fastlz_compress(const void* input, int length, void* output) -{ - return fastlz1_compress(input, length, output); - - // Fastlz2 doesn't really matter for us - // because of small block sizes - // return fastlz2_compress(input, length, output); -} - -int fastlz_decompress(const void* input, int length, void* output, int maxout) -{ - /* magic identifier for compression level */ - int level = ((*(const flzuint8*)input) >> 5) + 1; - - if(level == 1) - return fastlz1_decompress(input, length, output, maxout); - if(level == 2) - return fastlz2_decompress(input, length, output, maxout); - - /* unknown level, trigger error */ - return 0; -} - -int fastlz_compress_level(int level, const void* input, int length, void* output) -{ - if(level == 1) - return fastlz1_compress(input, length, output); - if(level == 2) - return fastlz2_compress(input, length, output); - - return 0; -} - -#else /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ - -static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output) -{ - const flzuint8* ip = (const flzuint8*) input; - const flzuint8* ip_bound = ip + length - 2; - const flzuint8* ip_limit = ip + length - 12; - flzuint8* op = (flzuint8*) output; - - const flzuint8* htab[HASH_SIZE]; - const flzuint8** hslot; - flzuint32 hval; - - flzuint32 copy; - - /* sanity check */ - if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4)) - { - if(length) - { - /* create literal copy only */ - *op++ = length-1; - ip_bound++; - while(ip <= ip_bound) - *op++ = *ip++; - return length+1; - } - else - return 0; - } - - /* initializes hash table */ - for (hslot = htab; hslot < htab + HASH_SIZE; hslot++) - *hslot = ip; - - /* we start with literal copy */ - copy = 2; - *op++ = MAX_COPY-1; - *op++ = *ip++; - *op++ = *ip++; - - /* main loop */ - while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - { - const flzuint8* ref; - flzuint32 distance; - - /* minimum match length */ - flzuint32 len = 3; - - /* comparison starting-point */ - const flzuint8* anchor = ip; - - /* check for a run */ -#if FASTLZ_LEVEL==2 - if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1)) - { - distance = 1; - ip += 3; - ref = anchor - 1 + 3; - goto match; - } -#endif - - /* find potential match */ - HASH_FUNCTION(hval,ip); - hslot = htab + hval; - ref = htab[hval]; - - /* calculate distance to the match */ - distance = anchor - ref; - - /* update hash table */ - *hslot = anchor; - - /* is this a match? check the first 3 bytes */ - if(distance==0 || (distance >= MAX_DISTANCE) || *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++) goto literal; - -#if FASTLZ_LEVEL==2 - /* far, needs at least 5-byte match */ - if(distance >= MAX_DISTANCE) - { - if(*ip++ != *ref++ || *ip++!= *ref++) - goto literal; - len += 2; - } - - match: -#endif - - /* last matched byte */ - ip = anchor + len; - - /* distance is biased */ - distance--; - - if(!distance) - { - /* zero distance means a run */ - flzuint8 x = ip[-1]; - while(ip < ip_bound) - if(*ref++ != x) break; else ip++; - } - else - for(;;) - { - /* safe because the outer check against ip limit */ - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - if(*ref++ != *ip++) break; - while(ip < ip_bound) - if(*ref++ != *ip++) break; - break; - } - - /* if we have copied something, adjust the copy count */ - if(copy) - /* copy is biased, '0' means 1 byte copy */ - *(op-copy-1) = copy-1; - else - /* back, to overwrite the copy count */ - op--; - - /* reset literal counter */ - copy = 0; - - /* length is biased, '1' means a match of 3 bytes */ - ip -= 3; - len = ip - anchor; - - /* encode the match */ -#if FASTLZ_LEVEL==2 - if(distance < MAX_DISTANCE) - { - if(len < 7) - { - *op++ = (len << 5) + (distance >> 8); - *op++ = (distance & 255); - } - else - { - *op++ = (7 << 5) + (distance >> 8); - for(len-=7; len >= 255; len-= 255) - *op++ = 255; - *op++ = len; - *op++ = (distance & 255); - } - } - else - { - /* far away, but not yet in the another galaxy... */ - if(len < 7) - { - distance -= MAX_DISTANCE; - *op++ = (len << 5) + 31; - *op++ = 255; - *op++ = distance >> 8; - *op++ = distance & 255; - } - else - { - distance -= MAX_DISTANCE; - *op++ = (7 << 5) + 31; - for(len-=7; len >= 255; len-= 255) - *op++ = 255; - *op++ = len; - *op++ = 255; - *op++ = distance >> 8; - *op++ = distance & 255; - } - } -#else - - if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2)) - while(len > MAX_LEN-2) - { - *op++ = (7 << 5) + (distance >> 8); - *op++ = MAX_LEN - 2 - 7 -2; - *op++ = (distance & 255); - len -= MAX_LEN-2; - } - - if(len < 7) - { - *op++ = (len << 5) + (distance >> 8); - *op++ = (distance & 255); - } - else - { - *op++ = (7 << 5) + (distance >> 8); - *op++ = len - 7; - *op++ = (distance & 255); - } -#endif - - /* update the hash at match boundary */ - HASH_FUNCTION(hval,ip); - htab[hval] = ip++; - HASH_FUNCTION(hval,ip); - htab[hval] = ip++; - - /* assuming literal copy */ - *op++ = MAX_COPY-1; - - continue; - - literal: - *op++ = *anchor++; - ip = anchor; - copy++; - if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) - { - copy = 0; - *op++ = MAX_COPY-1; - } - } - - /* left-over as literal copy */ - ip_bound++; - while(ip <= ip_bound) - { - *op++ = *ip++; - copy++; - if(copy == MAX_COPY) - { - copy = 0; - *op++ = MAX_COPY-1; - } - } - - /* if we have copied something, adjust the copy length */ - if(copy) - *(op-copy-1) = copy-1; - else - op--; - -#if FASTLZ_LEVEL==2 - /* marker for fastlz2 */ - *(flzuint8*)output |= (1 << 5); -#endif - - return op - (flzuint8*)output; -} - -static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout) -{ - const flzuint8* ip = (const flzuint8*) input; - const flzuint8* ip_limit = ip + length; - flzuint8* op = (flzuint8*) output; - flzuint8* op_limit = op + maxout; - flzuint32 ctrl = (*ip++) & 31; - int loop = 1; - - do - { - const flzuint8* ref = op; - flzuint32 len = ctrl >> 5; - flzuint32 ofs = (ctrl & 31) << 8; - - if(ctrl >= 32) - { -#if FASTLZ_LEVEL==2 - flzuint8 code; -#endif - len--; - ref -= ofs; - if (len == 7-1) -#if FASTLZ_LEVEL==1 - len += *ip++; - ref -= *ip++; -#else - do - { - code = *ip++; - len += code; - } while (code==255); - code = *ip++; - ref -= code; - - /* match from 16-bit distance */ - if(FASTLZ_UNEXPECT_CONDITIONAL(code==255)) - if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8))) - { - ofs = (*ip++) << 8; - ofs += *ip++; - ref = op - ofs - MAX_DISTANCE; - } -#endif - -#ifdef FASTLZ_SAFE - if (FASTLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) - return 0; - - if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output)) - return 0; -#endif - - if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - ctrl = *ip++; - else - loop = 0; - - if(ref == op) - { - /* optimize copy for a run */ - flzuint8 b = ref[-1]; - *op++ = b; - *op++ = b; - *op++ = b; - for(; len; --len) - *op++ = b; - } - else - { -#if !defined(FASTLZ_STRICT_ALIGN) - const flzuint16* p; - flzuint16* q; -#endif - /* copy from reference */ - ref--; - *op++ = *ref++; - *op++ = *ref++; - *op++ = *ref++; - -#if !defined(FASTLZ_STRICT_ALIGN) - /* copy a byte, so that now it's word aligned */ - if(len & 1) - { - *op++ = *ref++; - len--; - } - - /* copy 16-bit at once */ - q = (flzuint16*) op; - op += len; - p = (const flzuint16*) ref; - for(len>>=1; len > 4; len-=4) - { - *q++ = *p++; - *q++ = *p++; - *q++ = *p++; - *q++ = *p++; - } - for(; len; --len) - *q++ = *p++; -#else - for(; len; --len) - *op++ = *ref++; -#endif - } - } - else - { - ctrl++; -#ifdef FASTLZ_SAFE - if (FASTLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) - return 0; - if (FASTLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) - return 0; -#endif - - *op++ = *ip++; - for(--ctrl; ctrl; ctrl--) - *op++ = *ip++; - - loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit); - if(loop) - ctrl = *ip++; - } - } - while(FASTLZ_EXPECT_CONDITIONAL(loop)); - - return op - (flzuint8*)output; -} - -#endif /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ diff --git a/Modem/compression/fastlz.h b/Modem/compression/fastlz.h deleted file mode 100644 index f87bc7b..0000000 --- a/Modem/compression/fastlz.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - FastLZ - lightning-fast lossless compression library - - Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) - Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#ifndef FASTLZ_H -#define FASTLZ_H - -#define FASTLZ_VERSION 0x000100 - -#define FASTLZ_VERSION_MAJOR 0 -#define FASTLZ_VERSION_MINOR 0 -#define FASTLZ_VERSION_REVISION 0 - -#define FASTLZ_VERSION_STRING "0.1.0" - -#if defined (__cplusplus) -extern "C" { -#endif - -/** - Compress a block of data in the input buffer and returns the size of - compressed block. The size of input buffer is specified by length. The - minimum input buffer size is 16. - - The output buffer must be at least 5% larger than the input buffer - and can not be smaller than 66 bytes. - - If the input is not compressible, the return value might be larger than - length (input buffer size). - - The input buffer and the output buffer can not overlap. -*/ - -int fastlz_compress(const void* input, int length, void* output); - -/** - Decompress a block of compressed data and returns the size of the - decompressed block. If error occurs, e.g. the compressed data is - corrupted or the output buffer is not large enough, then 0 (zero) - will be returned instead. - - The input buffer and the output buffer can not overlap. - - Decompression is memory safe and guaranteed not to write the output buffer - more than what is specified in maxout. - */ - -int fastlz_decompress(const void* input, int length, void* output, int maxout); - -/** - Compress a block of data in the input buffer and returns the size of - compressed block. The size of input buffer is specified by length. The - minimum input buffer size is 16. - - The output buffer must be at least 5% larger than the input buffer - and can not be smaller than 66 bytes. - - If the input is not compressible, the return value might be larger than - length (input buffer size). - - The input buffer and the output buffer can not overlap. - - Compression level can be specified in parameter level. At the moment, - only level 1 and level 2 are supported. - Level 1 is the fastest compression and generally useful for short data. - Level 2 is slightly slower but it gives better compression ratio. - - Note that the compressed data, regardless of the level, can always be - decompressed using the function fastlz_decompress above. -*/ - -int fastlz_compress_level(int level, const void* input, int length, void* output); - -#if defined (__cplusplus) -} -#endif - -#endif /* FASTLZ_H */ diff --git a/Modem/compression/heatshrink_common.h b/Modem/compression/heatshrink_common.h new file mode 100644 index 0000000..27275d4 --- /dev/null +++ b/Modem/compression/heatshrink_common.h @@ -0,0 +1,19 @@ +#ifndef HEATSHRINK_H +#define HEATSHRINK_H + +#define HEATSHRINK_AUTHOR "Scott Vokes " + +/* Version 0.3.0 */ +#define HEATSHRINK_VERSION_MAJOR 0 +#define HEATSHRINK_VERSION_MINOR 3 +#define HEATSHRINK_VERSION_PATCH 0 + +#define HEATSHRINK_MIN_WINDOW_BITS 4 +#define HEATSHRINK_MAX_WINDOW_BITS 15 + +#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2 + +#define HEATSHRINK_LITERAL_MARKER 0x01 +#define HEATSHRINK_BACKREF_MARKER 0x00 + +#endif diff --git a/Modem/compression/heatshrink_config.h b/Modem/compression/heatshrink_config.h new file mode 100644 index 0000000..f618e2d --- /dev/null +++ b/Modem/compression/heatshrink_config.h @@ -0,0 +1,24 @@ +#ifndef HEATSHRINK_CONFIG_H +#define HEATSHRINK_CONFIG_H + +/* Should functionality assuming dynamic allocation be used? */ +#define HEATSHRINK_DYNAMIC_ALLOC 1 + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Optional replacement of malloc/free */ + #define HEATSHRINK_MALLOC(SZ) malloc(SZ) + #define HEATSHRINK_FREE(P, SZ) free(P) +#else + /* Required parameters for static configuration */ + #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 200 + #define HEATSHRINK_STATIC_WINDOW_BITS 8 + #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 +#endif + +/* Turn on logging for debugging. */ +#define HEATSHRINK_DEBUGGING_LOGS 0 + +/* Use indexing for faster compression. (This requires additional space.) */ +#define HEATSHRINK_USE_INDEX 0 + +#endif diff --git a/Modem/compression/heatshrink_decoder.c b/Modem/compression/heatshrink_decoder.c new file mode 100644 index 0000000..e6bd640 --- /dev/null +++ b/Modem/compression/heatshrink_decoder.c @@ -0,0 +1,376 @@ +#include +#include +#include "heatshrink_decoder.h" + +/* States for the polling state machine. */ +typedef enum { + HSDS_EMPTY, /* no input to process */ + HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */ + HSDS_YIELD_LITERAL, /* ready to yield literal byte */ + HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */ + HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */ + HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */ + HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */ + HSDS_YIELD_BACKREF, /* ready to yield back-reference */ + HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */ +} HSD_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "empty", + "input_available", + "yield_literal", + "backref_index", + "backref_count", + "yield_backref", + "check_for_more_input", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define NO_BITS ((uint32_t)-1) + +/* Forward references. */ +static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count); +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (input_buffer_size == 0) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 > window_sz2)) { + return NULL; + } + size_t buffers_sz = (1 << window_sz2) + input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz); + if (hsd == NULL) { return NULL; } + hsd->input_buffer_size = input_buffer_size; + hsd->window_sz2 = window_sz2; + hsd->lookahead_sz2 = lookahead_sz2; + heatshrink_decoder_reset(hsd); + LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n", + sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size); + return hsd; +} + +void heatshrink_decoder_free(heatshrink_decoder *hsd) { + size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + HEATSHRINK_FREE(hsd, sz); + (void)sz; /* may not be used by free */ +} +#endif + +void heatshrink_decoder_reset(heatshrink_decoder *hsd) { + size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd); + size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd); + memset(hsd->buffers, 0, buf_sz + input_sz); + hsd->state = HSDS_EMPTY; + hsd->input_size = 0; + hsd->input_index = 0; + hsd->bit_index = 0x00; + hsd->current_byte = 0x00; + hsd->output_count = 0; + hsd->output_index = 0; + hsd->head_index = 0; + hsd->bit_accumulator = 0x00000000; +} + +/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSDR_SINK_ERROR_NULL; + } + + size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size; + if (rem == 0) { + *input_size = 0; + return HSDR_SINK_FULL; + } + + size = rem < size ? rem : size; + LOG("-- sinking %zd bytes\n", size); + /* copy into input buffer (at head of buffers) */ + memcpy(&hsd->buffers[hsd->input_size], in_buf, size); + hsd->input_size += size; + if (hsd->state == HSDS_EMPTY) { + hsd->state = HSDS_INPUT_AVAILABLE; + hsd->input_index = 0; + } + *input_size = size; + return HSDR_SINK_OK; +} + + +/***************** + * Decompression * + *****************/ + +#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD)) +#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD)) + +// States +static HSD_state st_input_available(heatshrink_decoder *hsd); +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd); +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_check_for_input(heatshrink_decoder *hsd); + +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSDR_POLL_ERROR_NULL; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- poll, state is %d (%s), input_size %d\n", + hsd->state, state_names[hsd->state], hsd->input_size); + uint8_t in_state = hsd->state; + switch (in_state) { + case HSDS_EMPTY: + return HSDR_POLL_EMPTY; + case HSDS_INPUT_AVAILABLE: + hsd->state = st_input_available(hsd); + break; + case HSDS_YIELD_LITERAL: + hsd->state = st_yield_literal(hsd, &oi); + break; + case HSDS_BACKREF_INDEX_MSB: + hsd->state = st_backref_index_msb(hsd); + break; + case HSDS_BACKREF_INDEX_LSB: + hsd->state = st_backref_index_lsb(hsd); + break; + case HSDS_BACKREF_COUNT_MSB: + hsd->state = st_backref_count_msb(hsd); + break; + case HSDS_BACKREF_COUNT_LSB: + hsd->state = st_backref_count_lsb(hsd); + break; + case HSDS_YIELD_BACKREF: + hsd->state = st_yield_backref(hsd, &oi); + break; + case HSDS_CHECK_FOR_MORE_INPUT: + hsd->state = st_check_for_input(hsd); + break; + default: + return HSDR_POLL_ERROR_UNKNOWN; + } + + /* If the current state cannot advance, check if input or output + * buffer are exhausted. */ + if (hsd->state == in_state) { + if (*output_size == out_buf_size) { return HSDR_POLL_MORE; } + return HSDR_POLL_EMPTY; + } + } +} + +static HSD_state st_input_available(heatshrink_decoder *hsd) { + uint32_t bits = get_bits(hsd, 1); // get tag bit + if (bits) { + return HSDS_YIELD_LITERAL; + } else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) { + return HSDS_BACKREF_INDEX_MSB; + } else { + hsd->output_index = 0; + return HSDS_BACKREF_INDEX_LSB; + } +} + +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi) { + /* Emit a repeated section from the window buffer, and add it (again) + * to the window buffer. (Note that the repetition can include + * itself.)*/ + if (*oi->output_size < oi->buf_size) { + uint32_t byte = get_bits(hsd, 8); + if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */ + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint8_t c = byte & 0xFF; + LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.'); + buf[hsd->head_index++ & mask] = c; + push_byte(hsd, oi, c); + return HSDS_CHECK_FOR_MORE_INPUT; + } else { + return HSDS_YIELD_LITERAL; + } +} + +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + ASSERT(bit_ct > 8); + uint32_t bits = get_bits(hsd, bit_ct - 8); + LOG("-- backref index (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; } + hsd->output_index = bits << 8; + return HSDS_BACKREF_INDEX_LSB; +} + +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8); + LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; } + hsd->output_index |= bits; + hsd->output_index++; + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + hsd->output_count = 0; + return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + ASSERT(br_bit_ct > 8); + uint32_t bits = get_bits(hsd, br_bit_ct - 8); + LOG("-- backref count (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; } + hsd->output_count = bits << 8; + return HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8); + LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; } + hsd->output_count |= bits; + hsd->output_count++; + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi) { + size_t count = oi->buf_size - *oi->output_size; + if (count > 0) { + if (hsd->output_count < count) count = hsd->output_count; + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint16_t neg_offset = hsd->output_index; + LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset); + ASSERT(neg_offset < mask + 1); + ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd)); + + for (size_t i=0; ihead_index - neg_offset) & mask]; + push_byte(hsd, oi, c); + buf[hsd->head_index & mask] = c; + hsd->head_index++; + LOG(" -- ++ 0x%02x\n", c); + } + hsd->output_count -= count; + if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; } + } + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_check_for_input(heatshrink_decoder *hsd) { + return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE; +} + +/* Get the next COUNT bits from the input buffer, saving incremental progress. + * Returns NO_BITS on end of input, or if more than 31 bits are requested. */ +static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) { + if (count > 31) { return NO_BITS; } + LOG("-- popping %u bit(s)\n", count); + + /* If we aren't able to get COUNT bits, suspend immediately, because we + * don't track how many bits of COUNT we've accumulated before suspend. */ + if (hsd->input_size == 0) { + if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; } + } + + for (int i = 0; i < count; i++) { + if (hsd->bit_index == 0x00) { + if (hsd->input_size == 0) { + LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n", + hsd->bit_accumulator, hsd->bit_accumulator); + return NO_BITS; + } + hsd->current_byte = hsd->buffers[hsd->input_index++]; + LOG(" -- pulled byte 0x%02x\n", hsd->current_byte); + if (hsd->input_index == hsd->input_size) { + hsd->input_index = 0; /* input is exhausted */ + hsd->input_size = 0; + } + hsd->bit_index = 0x80; + } + hsd->bit_accumulator <<= 1; + if (hsd->current_byte & hsd->bit_index) { + hsd->bit_accumulator |= 0x01; + if (0) { + LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n", + hsd->bit_accumulator, hsd->bit_index); + } + } else { + if (0) { + LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n", + hsd->bit_accumulator, hsd->bit_index); + } + } + hsd->bit_index >>= 1; + } + + uint32_t res = 0; + res = hsd->bit_accumulator; + hsd->bit_accumulator = 0x00000000; + if (count > 1) { LOG(" -- accumulated %08x\n", res); } + return res; +} + +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) { + if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; } + switch (hsd->state) { + case HSDS_EMPTY: + return HSDR_FINISH_DONE; + + /* If we want to finish with no input, but are in these states, it's + * because the 0-bit padding to the last byte looks like a backref + * marker bit followed by all 0s for index and count bits. */ + case HSDS_BACKREF_INDEX_LSB: + case HSDS_BACKREF_INDEX_MSB: + case HSDS_BACKREF_COUNT_LSB: + case HSDS_BACKREF_COUNT_MSB: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + /* fall through */ + default: + return HSDR_FINISH_MORE; + } +} + +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) { + LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.'); + oi->buf[(*oi->output_size)++] = byte; + (void)hsd; +} diff --git a/Modem/compression/heatshrink_decoder.h b/Modem/compression/heatshrink_decoder.h new file mode 100644 index 0000000..c1eb144 --- /dev/null +++ b/Modem/compression/heatshrink_decoder.h @@ -0,0 +1,101 @@ +#ifndef HEATSHRINK_DECODER_H +#define HEATSHRINK_DECODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSDR_SINK_OK, /* data sunk, ready to poll */ + HSDR_SINK_FULL, /* out of space in internal buffer */ + HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ +} HSD_sink_res; + +typedef enum { + HSDR_POLL_EMPTY, /* input exhausted */ + HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ + HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ + HSDR_POLL_ERROR_UNKNOWN=-2, +} HSD_poll_res; + +typedef enum { + HSDR_FINISH_DONE, /* output is done */ + HSDR_FINISH_MORE, /* more output remains */ + HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ +} HSD_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ + ((BUF)->input_buffer_size) +#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ + ((BUF)->window_sz2) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + ((BUF)->lookahead_sz2) +#else +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ + HEATSHRINK_STATIC_INPUT_BUFFER_SIZE +#define HEATSHRINK_DECODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t input_index; /* offset to next unprocessed input byte */ + uint16_t output_count; /* how many bytes to output */ + uint16_t output_index; /* index for bytes to output */ + uint16_t head_index; /* head of window buffer */ + uint16_t bit_accumulator; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of input */ + uint8_t bit_index; /* current bit index */ + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Fields that are only used if dynamically allocated. */ + uint8_t window_sz2; /* window buffer bits */ + uint8_t lookahead_sz2; /* lookahead bits */ + uint16_t input_buffer_size; /* input buffer size */ + + /* Input buffer, then expansion window buffer */ + uint8_t buffers[]; +#else + /* Input buffer, then expansion window buffer */ + uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) + + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; +#endif +} heatshrink_decoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, + * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead + * size of 2^lookahead_sz2. (The window buffer and lookahead sizes + * must match the settings used when the data was compressed.) + * Returns NULL on error. */ +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); + +/* Free a decoder. */ +void heatshrink_decoder_free(heatshrink_decoder *hsd); +#endif + +/* Reset a decoder. */ +void heatshrink_decoder_reset(heatshrink_decoder *hsd); + +/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to + * indicate how many bytes were actually sunk (in case a buffer was filled). */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the dencoder that the input stream is finished. + * If the return value is HSDR_FINISH_MORE, there is still more output, so + * call heatshrink_decoder_poll and repeat. */ +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); + +#endif diff --git a/Modem/compression/heatshrink_encoder.c b/Modem/compression/heatshrink_encoder.c new file mode 100644 index 0000000..6772c8a --- /dev/null +++ b/Modem/compression/heatshrink_encoder.c @@ -0,0 +1,641 @@ +#include +#include +#include +#include "heatshrink_encoder.h" + +#include "cfg/debug.h" // Debug configuration from BertOS + +typedef enum { + HSES_NOT_FULL, /* input buffer not full enough */ + HSES_FILLED, /* buffer is full */ + HSES_SEARCH, /* searching for patterns */ + HSES_YIELD_TAG_BIT, /* yield tag bit */ + HSES_YIELD_LITERAL, /* emit literal byte */ + HSES_YIELD_BR_INDEX, /* yielding backref index */ + HSES_YIELD_BR_LENGTH, /* yielding backref length */ + HSES_SAVE_BACKLOG, /* copying buffer to backlog */ + HSES_FLUSH_BITS, /* flush bit buffer */ + HSES_DONE, /* done */ +} HSE_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "not_full", + "filled", + "search", + "yield_tag_bit", + "yield_literal", + "yield_br_index", + "yield_br_length", + "save_backlog", + "flush_bits", + "done", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +// Encoder flags +enum { + FLAG_IS_FINISHING = 0x01, + FLAG_HAS_LITERAL = 0x02, + FLAG_ON_FINAL_LITERAL = 0x04, + FLAG_BACKLOG_IS_PARTIAL = 0x08, + FLAG_BACKLOG_IS_FILLED = 0x10, +}; + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define MATCH_NOT_FOUND ((uint16_t)-1) + +static uint16_t get_input_offset(heatshrink_encoder *hse); +static uint16_t get_input_buffer_size(heatshrink_encoder *hse); +static uint16_t get_lookahead_size(heatshrink_encoder *hse); +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag); +static int can_take_byte(output_info *oi); +static int is_finishing(heatshrink_encoder *hse); +static int backlog_is_partial(heatshrink_encoder *hse); +static int backlog_is_filled(heatshrink_encoder *hse); +static int on_final_literal(heatshrink_encoder *hse); +static void save_backlog(heatshrink_encoder *hse); +static int has_literal(heatshrink_encoder *hse); + +/* Push COUNT (max 8) bits to the output buffer, which has room. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi); +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi); +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 > window_sz2)) { + return NULL; + } + + /* Note: 2 * the window size is used because the buffer needs to fit + * (1 << window_sz2) bytes for the current input, and an additional + * (1 << window_sz2) bytes for the previous buffer of input, which + * will be scanned for useful backreferences. */ + size_t buf_sz = (2 << window_sz2); + + kprintf("Trying to allocate: %d\n", buf_sz); + heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz); + if (hse == NULL) { return NULL; } + hse->window_sz2 = window_sz2; + hse->lookahead_sz2 = lookahead_sz2; + heatshrink_encoder_reset(hse); + +#if HEATSHRINK_USE_INDEX + size_t index_sz = buf_sz*sizeof(uint16_t); + hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index)); + if (hse->search_index == NULL) { + HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz); + return NULL; + } + hse->search_index->size = index_sz; +#endif + + LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n", + buf_sz, get_input_buffer_size(hse)); + return hse; +} + +void heatshrink_encoder_free(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); +#if HEATSHRINK_USE_INDEX + size_t index_sz = sizeof(struct hs_index) + hse->search_index->size; + HEATSHRINK_FREE(hse->search_index, index_sz); + (void)index_sz; +#endif + HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz); + (void)buf_sz; +} +#endif + +void heatshrink_encoder_reset(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + memset(hse->buffer, 0, buf_sz); + hse->input_size = 0; + hse->state = HSES_NOT_FULL; + hse->match_scan_index = 0; + hse->flags = 0; + hse->bit_index = 0x80; + hse->current_byte = 0x00; + hse->match_length = 0; + + hse->outgoing_bits = 0x0000; + hse->outgoing_bits_count = 0; + + #ifdef LOOP_DETECT + hse->loop_detect = (uint32_t)-1; + #endif +} + +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSER_SINK_ERROR_NULL; + } + + /* Sinking more content after saying the content is done, tsk tsk */ + if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; } + + /* Sinking more content before processing is done */ + if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; } + + uint16_t write_offset = get_input_offset(hse) + hse->input_size; + uint16_t ibs = get_input_buffer_size(hse); + uint16_t rem = ibs - hse->input_size; + uint16_t cp_sz = rem < size ? rem : size; + + memcpy(&hse->buffer[write_offset], in_buf, cp_sz); + *input_size = cp_sz; + hse->input_size += cp_sz; + + LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n", + cp_sz, size, write_offset, hse->input_size); + if (cp_sz == rem) { + LOG("-- internal buffer is now full\n"); + hse->state = HSES_FILLED; + } + + return HSER_SINK_OK; +} + + +/*************** + * Compression * + ***************/ + +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length); +static void do_indexing(heatshrink_encoder *hse); + +static HSE_state st_step_search(heatshrink_encoder *hse); +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_save_backlog(heatshrink_encoder *hse); +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi); + +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSER_POLL_ERROR_NULL; + } + if (out_buf_size == 0) { + LOG("-- MISUSE: output buffer size is 0\n"); + return HSER_POLL_ERROR_MISUSE; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- polling, state %u (%s), flags 0x%02x\n", + hse->state, state_names[hse->state], hse->flags); + + uint8_t in_state = hse->state; + switch (in_state) { + case HSES_NOT_FULL: + return HSER_POLL_EMPTY; + case HSES_FILLED: + do_indexing(hse); + hse->state = HSES_SEARCH; + break; + case HSES_SEARCH: + hse->state = st_step_search(hse); + break; + case HSES_YIELD_TAG_BIT: + hse->state = st_yield_tag_bit(hse, &oi); + break; + case HSES_YIELD_LITERAL: + hse->state = st_yield_literal(hse, &oi); + break; + case HSES_YIELD_BR_INDEX: + hse->state = st_yield_br_index(hse, &oi); + break; + case HSES_YIELD_BR_LENGTH: + hse->state = st_yield_br_length(hse, &oi); + break; + case HSES_SAVE_BACKLOG: + hse->state = st_save_backlog(hse); + break; + case HSES_FLUSH_BITS: + hse->state = st_flush_bit_buffer(hse, &oi); + case HSES_DONE: + return HSER_POLL_EMPTY; + default: + LOG("-- bad state %s\n", state_names[hse->state]); + return HSER_POLL_ERROR_MISUSE; + } + + if (hse->state == in_state) { + /* Check if output buffer is exhausted. */ + if (*output_size == out_buf_size) return HSER_POLL_MORE; + } + } +} + +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) { + if (hse == NULL) { return HSER_FINISH_ERROR_NULL; } + LOG("-- setting is_finishing flag\n"); + hse->flags |= FLAG_IS_FINISHING; + if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; } + return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE; +} + +static HSE_state st_step_search(heatshrink_encoder *hse) { + uint16_t window_length = get_input_buffer_size(hse); + uint16_t lookahead_sz = get_lookahead_size(hse); + uint16_t msi = hse->match_scan_index; + LOG("## step_search, scan @ +%d (%d/%d), input size %d\n", + msi, hse->input_size + msi, 2*window_length, hse->input_size); + + bool fin = is_finishing(hse); + if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) { + /* Current search buffer is exhausted, copy it into the + * backlog and await more input. */ + LOG("-- end of search @ %d, saving backlog\n", msi); + return HSES_SAVE_BACKLOG; + } + + uint16_t input_offset = get_input_offset(hse); + uint16_t end = input_offset + msi; + + uint16_t start = 0; + if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */ + start = end - window_length + 1; + } else if (backlog_is_partial(hse)) { /* clamp to available data */ + start = end - window_length + 1; + if (start < lookahead_sz) { start = lookahead_sz; } + } else { /* only scan available input */ + start = input_offset; + } + + uint16_t max_possible = lookahead_sz; + if (hse->input_size - msi < lookahead_sz) { + max_possible = hse->input_size - msi; + } + + uint16_t match_length = 0; + uint16_t match_pos = find_longest_match(hse, + start, end, max_possible, &match_length); + + if (match_pos == MATCH_NOT_FOUND) { + LOG("ss Match not found\n"); + hse->match_scan_index++; + hse->flags |= FLAG_HAS_LITERAL; + hse->match_length = 0; + return HSES_YIELD_TAG_BIT; + } else { + LOG("ss Found match of %d bytes at %d\n", match_length, match_pos); + hse->match_pos = match_pos; + hse->match_length = match_length; + ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/); + + return HSES_YIELD_TAG_BIT; + } +} + +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + if (hse->match_length == 0) { + add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER); + return HSES_YIELD_LITERAL; + } else { + add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER); + hse->outgoing_bits = hse->match_pos - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse); + return HSES_YIELD_BR_INDEX; + } + } else { + return HSES_YIELD_TAG_BIT; /* output is full, continue */ + } +} + +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + push_literal_byte(hse, oi); + hse->flags &= ~FLAG_HAS_LITERAL; + if (on_final_literal(hse)) { return HSES_FLUSH_BITS; } + return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH; + } else { + return HSES_YIELD_LITERAL; + } +} + +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref index %u\n", hse->match_pos); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_INDEX; /* continue */ + } else { + hse->outgoing_bits = hse->match_length - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse); + return HSES_YIELD_BR_LENGTH; /* done */ + } + } else { + return HSES_YIELD_BR_INDEX; /* continue */ + } +} + +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref length %u\n", hse->match_length); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_LENGTH; + } else { + hse->match_scan_index += hse->match_length; + hse->match_length = 0; + return HSES_SEARCH; + } + } else { + return HSES_YIELD_BR_LENGTH; + } +} + +static HSE_state st_save_backlog(heatshrink_encoder *hse) { + if (is_finishing(hse)) { + /* copy remaining literal (if necessary) */ + if (has_literal(hse)) { + hse->flags |= FLAG_ON_FINAL_LITERAL; + return HSES_YIELD_TAG_BIT; + } else { + return HSES_FLUSH_BITS; + } + } else { + LOG("-- saving backlog\n"); + save_backlog(hse); + return HSES_NOT_FULL; + } +} + +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi) { + if (hse->bit_index == 0x80) { + LOG("-- done!\n"); + return HSES_DONE; + } else if (can_take_byte(oi)) { + LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index); + oi->buf[(*oi->output_size)++] = hse->current_byte; + LOG("-- done!\n"); + return HSES_DONE; + } else { + return HSES_FLUSH_BITS; + } +} + +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) { + LOG("-- adding tag bit: %d\n", tag); + push_bits(hse, 1, tag, oi); +} + +static uint16_t get_input_offset(heatshrink_encoder *hse) { + return get_input_buffer_size(hse); +} + +static uint16_t get_input_buffer_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + (void)hse; +} + +static uint16_t get_lookahead_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); + (void)hse; +} + +static void do_indexing(heatshrink_encoder *hse) { +#if HEATSHRINK_USE_INDEX + /* Build an index array I that contains flattened linked lists + * for the previous instances of every byte in the buffer. + * + * For example, if buf[200] == 'x', then index[200] will either + * be an offset i such that buf[i] == 'x', or a negative offset + * to indicate end-of-list. This significantly speeds up matching, + * while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM. + * + * Future optimization options: + * 1. Since any negative value represents end-of-list, the other + * 15 bits could be used to improve the index dynamically. + * + * 2. Likewise, the last lookahead_sz bytes of the index will + * not be usable, so temporary data could be stored there to + * dynamically improve the index. + * */ + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + uint16_t last[256]; + memset(last, 0xFF, sizeof(last)); + + uint8_t * const data = hse->buffer; + int16_t * const index = hsi->index; + + const uint16_t input_offset = get_input_offset(hse); + const uint16_t end = input_offset + hse->input_size; + + for (uint16_t i=0; iflags & FLAG_IS_FINISHING; +} + +static int backlog_is_partial(heatshrink_encoder *hse) { + return hse->flags & FLAG_BACKLOG_IS_PARTIAL; +} + +static int backlog_is_filled(heatshrink_encoder *hse) { + return hse->flags & FLAG_BACKLOG_IS_FILLED; +} + +static int on_final_literal(heatshrink_encoder *hse) { + return hse->flags & FLAG_ON_FINAL_LITERAL; +} + +static int has_literal(heatshrink_encoder *hse) { + return (hse->flags & FLAG_HAS_LITERAL); +} + +static int can_take_byte(output_info *oi) { + return *oi->output_size < oi->buf_size; +} + +/* Return the longest match for the bytes at buf[end:end+maxlen] between + * buf[start] and buf[end-1]. If no match is found, return -1. */ +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length) { + LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n", + end, end + maxlen, start, end + maxlen - 1, maxlen); + uint8_t *buf = hse->buffer; + + uint16_t match_maxlen = 0; + uint16_t match_index = MATCH_NOT_FOUND; + const uint16_t break_even_point = 3; + uint16_t len = 0; + uint8_t * const needlepoint = &buf[end]; +#if HEATSHRINK_USE_INDEX + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + int16_t pos = hsi->index[end]; + + while (pos >= start) { + uint8_t * const pospoint = &buf[pos]; + len = 0; + + for (len = 1; len < maxlen; len++) { + if (pospoint[len] != needlepoint[len]) break; + } + + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* won't find better */ + } + pos = hsi->index[pos]; + } +#else + for (int16_t pos=end - 1; pos >= start; pos--) { + for (len=0; len cmp buf[%d] == 0x%02x against %02x (start %u)\n", + pos + len, buf[pos + len], needlepoint[len], start); + } + if (buf[pos + len] != needlepoint[len]) { break; } + } + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* don't keep searching */ + } + } +#endif + + if (match_maxlen >= break_even_point) { + LOG("-- best match: %u bytes at -%u\n", + match_maxlen, end - match_index); + *match_length = match_maxlen; + return end - match_index; + } + LOG("-- none found\n"); + return MATCH_NOT_FOUND; +} + +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) { + uint8_t count = 0; + uint8_t bits = 0; + if (hse->outgoing_bits_count > 8) { + count = 8; + bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8); + } else { + count = hse->outgoing_bits_count; + bits = hse->outgoing_bits; + } + + if (count > 0) { + LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits); + push_bits(hse, count, bits, oi); + hse->outgoing_bits_count -= count; + } + return count; +} + +/* Push COUNT (max 8) bits to the output buffer, which has room. + * Bytes are set from the lowest bits, up. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi) { + ASSERT(count <= 8); + LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits); + + /* If adding a whole byte and at the start of a new output byte, + * just push it through whole and skip the bit IO loop. */ + if (count == 8 && hse->bit_index == 0x80) { + oi->buf[(*oi->output_size)++] = bits; + } else { + for (int i=count - 1; i>=0; i--) { + bool bit = bits & (1 << i); + if (bit) { hse->current_byte |= hse->bit_index; } + if (0) { + LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n", + bit ? 1 : 0, hse->bit_index, hse->current_byte); + } + hse->bit_index >>= 1; + if (hse->bit_index == 0x00) { + hse->bit_index = 0x80; + LOG(" > pushing byte 0x%02x\n", hse->current_byte); + oi->buf[(*oi->output_size)++] = hse->current_byte; + hse->current_byte = 0x00; + } + } + } +} + +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) { + uint16_t processed_offset = hse->match_scan_index - 1; + uint16_t input_offset = get_input_offset(hse) + processed_offset; + uint8_t c = hse->buffer[input_offset]; + LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n", + c, isprint(c) ? c : '.', input_offset); + push_bits(hse, 8, c, oi); +} + +static void save_backlog(heatshrink_encoder *hse) { + size_t input_buf_sz = get_input_buffer_size(hse); + + uint16_t msi = hse->match_scan_index; + + /* Copy processed data to beginning of buffer, so it can be + * used for future matches. Don't bother checking whether the + * input is less than the maximum size, because if it isn't, + * we're done anyway. */ + uint16_t rem = input_buf_sz - msi; // unprocessed bytes + uint16_t shift_sz = input_buf_sz + rem; + + memmove(&hse->buffer[0], + &hse->buffer[input_buf_sz - rem], + shift_sz); + + if (backlog_is_partial(hse)) { + /* The whole backlog is filled in now, so include it in scans. */ + hse->flags |= FLAG_BACKLOG_IS_FILLED; + } else { + /* Include backlog, except for the first lookahead_sz bytes, which + * are still undefined. */ + hse->flags |= FLAG_BACKLOG_IS_PARTIAL; + } + hse->match_scan_index = 0; + hse->input_size -= input_buf_sz - rem; +} diff --git a/Modem/compression/heatshrink_encoder.h b/Modem/compression/heatshrink_encoder.h new file mode 100644 index 0000000..18c1773 --- /dev/null +++ b/Modem/compression/heatshrink_encoder.h @@ -0,0 +1,109 @@ +#ifndef HEATSHRINK_ENCODER_H +#define HEATSHRINK_ENCODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSER_SINK_OK, /* data sunk into input buffer */ + HSER_SINK_ERROR_NULL=-1, /* NULL argument */ + HSER_SINK_ERROR_MISUSE=-2, /* API misuse */ +} HSE_sink_res; + +typedef enum { + HSER_POLL_EMPTY, /* input exhausted */ + HSER_POLL_MORE, /* poll again for more output */ + HSER_POLL_ERROR_NULL=-1, /* NULL argument */ + HSER_POLL_ERROR_MISUSE=-2, /* API misuse */ +} HSE_poll_res; + +typedef enum { + HSER_FINISH_DONE, /* encoding is complete */ + HSER_FINISH_MORE, /* more output remaining; use poll */ + HSER_FINISH_ERROR_NULL=-1, /* NULL argument */ +} HSE_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \ + ((HSE)->window_sz2) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \ + ((HSE)->lookahead_sz2) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + ((HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[]; +}; +#else +#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + (&(HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS]; +}; +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t match_scan_index; + uint16_t match_length; + uint16_t match_pos; + uint16_t outgoing_bits; /* enqueued outgoing bits */ + uint8_t outgoing_bits_count; + uint8_t flags; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of output */ + uint8_t bit_index; /* current bit index */ +#if HEATSHRINK_DYNAMIC_ALLOC + uint8_t window_sz2; /* 2^n size of window */ + uint8_t lookahead_sz2; /* 2^n size of lookahead */ +#if HEATSHRINK_USE_INDEX + struct hs_index *search_index; +#endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[]; +#else + #if HEATSHRINK_USE_INDEX + struct hs_index search_index; + #endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)]; +#endif +} heatshrink_encoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a new encoder struct and its buffers. + * Returns NULL on error. */ +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2); + +/* Free an encoder. */ +void heatshrink_encoder_free(heatshrink_encoder *hse); +#endif + +/* Reset an encoder. */ +void heatshrink_encoder_reset(heatshrink_encoder *hse); + +/* Sink up to SIZE bytes from IN_BUF into the encoder. + * INPUT_SIZE is set to the number of bytes actually sunk (in case a + * buffer was filled.). */ +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the encoder that the input stream is finished. + * If the return value is HSER_FINISH_MORE, there is still more output, so + * call heatshrink_encoder_poll and repeat. */ +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse); + +#endif diff --git a/Modem/compression/lzfx.c b/Modem/compression/lzfx.c deleted file mode 100644 index 9bf7cd3..0000000 --- a/Modem/compression/lzfx.c +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Copyright (c) 2009 Andrew Collette - * http://lzfx.googlecode.com - * - * Implements an LZF-compatible compressor/decompressor based on the liblzf - * codebase written by Marc Lehmann. This code is released under the BSD - * license. License and original copyright statement follow. - * - * - * Copyright (c) 2000-2008 Marc Alexander Lehmann - * - * Redistribution and use in source and binary forms, with or without modifica- - * tion, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- - * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- - * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- - * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "lzfx.h" - -#define LZFX_HSIZE (1 << (LZFX_HLOG)) - -/* We need this for memset */ -# include - -// #if __GNUC__ >= 3 && !DISABLE_EXPECT -// # define fx_expect_false(expr) __builtin_expect((expr) != 0, 0) -// # define fx_expect_true(expr) __builtin_expect((expr) != 0, 1) -// #else -# define fx_expect_false(expr) (expr) -# define fx_expect_true(expr) (expr) -//#endif - -typedef unsigned char u8; -typedef const u8 *LZSTATE[LZFX_HSIZE]; - -/* Define the hash function */ -#define LZFX_FRST(p) (((p[0]) << 8) | p[1]) -#define LZFX_NEXT(v,p) (((v) << 8) | p[2]) -#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1)) - -/* These cannot be changed, as they are related to the compressed format. */ -#define LZFX_MAX_LIT (1 << 5) -#define LZFX_MAX_OFF (1 << 13) -#define LZFX_MAX_REF ((1 << 8) + (1 << 3)) - -static -int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen); - -/* Compressed format - - There are two kinds of structures in LZF/LZFX: literal runs and back - references. The length of a literal run is encoded as L - 1, as it must - contain at least one byte. Literals are encoded as follows: - - 000LLLLL - - Back references are encoded as follows. The smallest possible encoded - length value is 1, as otherwise the control byte would be recognized as - a literal run. Since at least three bytes must match for a back reference - to be inserted, the length is encoded as L - 2 instead of L - 1. The - offset (distance to the desired data in the output buffer) is encoded as - o - 1, as all offsets are at least 1. The binary format is: - - LLLooooo oooooooo for backrefs of real length < 9 (1 <= L < 7) - 111ooooo LLLLLLLL oooooooo for backrefs of real length >= 9 (L > 7) -*/ -#include -int lzfx_compress(const void *const ibuf, const unsigned int ilen, - void *obuf, unsigned int *const olen){ - - /* Hash table; an array of u8*'s which point - to various locations in the input buffer */ - const u8 *htab[LZFX_HSIZE]; - - const u8 **hslot; /* Pointer to entry in hash table */ - unsigned int hval; /* Hash value generated by macros above */ - const u8 *ref; /* Pointer to candidate match location in input */ - - const u8 *ip = (const u8 *)ibuf; - const u8 *const in_end = ip + ilen; - - u8 *op = (u8 *)obuf; - const u8 *const out_end = (olen == NULL ? NULL : op + *olen); - - int lit; /* # of bytes in current literal run */ - -#if defined (WIN32) && defined (_M_X64) - unsigned _int64 off; /* workaround for missing POSIX compliance */ -#else - unsigned long off; -#endif - - if(olen == NULL) return LZFX_EARGS; - if(ibuf == NULL){ - if(ilen != 0) return LZFX_EARGS; - *olen = 0; - return 0; - } - if(obuf == NULL){ - if(olen != 0) return LZFX_EARGS; - return lzfx_getsize(ibuf, ilen, olen); - } - - memset(htab, 0, sizeof(htab)); - - /* Start a literal run. Whenever we do this the output pointer is - advanced because the current byte will hold the encoded length. */ - lit = 0; op++; - - hval = LZFX_FRST(ip); - - while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */ - - hval = LZFX_NEXT(hval, ip); - hslot = htab + LZFX_IDX(hval); - - ref = *hslot; *hslot = ip; - - if( ref < ip - && (off = ip - ref - 1) < LZFX_MAX_OFF - && ip + 4 < in_end /* Backref takes up to 3 bytes, so don't bother */ - && ref > (u8 *)ibuf - && ref[0] == ip[0] - && ref[1] == ip[1] - && ref[2] == ip[2] ) { - - unsigned int len = 3; /* We already know 3 bytes match */ - const unsigned int maxlen = in_end - ip - 2 > LZFX_MAX_REF ? - LZFX_MAX_REF : in_end - ip - 2; - - /* lit == 0: op + 3 must be < out_end (because we undo the run) - lit != 0: op + 3 + 1 must be < out_end */ - if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) - return LZFX_ESIZE; - - op [- lit - 1] = lit - 1; /* Terminate literal run */ - op -= !lit; /* Undo run if length is zero */ - - /* Start checking at the fourth byte */ - while (len < maxlen && ref[len] == ip[len]) - len++; - - len -= 2; /* We encode the length as #octets - 2 */ - - /* Format 1: [LLLooooo oooooooo] */ - if (len < 7) { - *op++ = (off >> 8) + (len << 5); - *op++ = off; - - /* Format 2: [111ooooo LLLLLLLL oooooooo] */ - } else { - *op++ = (off >> 8) + (7 << 5); - *op++ = len - 7; - *op++ = off; - } - - lit = 0; op++; - - ip += len + 1; /* ip = initial ip + #octets -1 */ - - if (fx_expect_false (ip + 3 >= in_end)){ - ip++; /* Code following expects exit at bottom of loop */ - break; - } - - hval = LZFX_FRST (ip); - hval = LZFX_NEXT (hval, ip); - htab[LZFX_IDX (hval)] = ip; - - ip++; /* ip = initial ip + #octets */ - - } else { - /* Keep copying literal bytes */ - - if (fx_expect_false (op >= out_end)) return LZFX_ESIZE; - - lit++; *op++ = *ip++; - - if (fx_expect_false (lit == LZFX_MAX_LIT)) { - op [- lit - 1] = lit - 1; /* stop run */ - lit = 0; op++; /* start run */ - } - - } /* if() found match in htab */ - - } /* while(ip < ilen -2) */ - - /* At most 3 bytes remain in input. We therefore need 4 bytes available - in the output buffer to store them (3 data + ctrl byte).*/ - if (op + 3 > out_end) return LZFX_ESIZE; - - while (ip < in_end) { - - lit++; *op++ = *ip++; - - if (fx_expect_false (lit == LZFX_MAX_LIT)){ - op [- lit - 1] = lit - 1; - lit = 0; op++; - } - } - - op [- lit - 1] = lit - 1; - op -= !lit; - - *olen = op - (u8 *)obuf; - return 0; -} - -/* Decompressor */ -int lzfx_decompress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen){ - - u8 const *ip = (const u8 *)ibuf; - u8 const *const in_end = ip + ilen; - u8 *op = (u8 *)obuf; - u8 const *const out_end = (olen == NULL ? NULL : op + *olen); - - unsigned int remain_len = 0; - int rc; - - if(olen == NULL) return LZFX_EARGS; - if(ibuf == NULL){ - if(ilen != 0) return LZFX_EARGS; - *olen = 0; - return 0; - } - if(obuf == NULL){ - if(olen != 0) return LZFX_EARGS; - return lzfx_getsize(ibuf, ilen, olen); - } - - do { - unsigned int ctrl = *ip++; - - /* Format 000LLLLL: a literal byte string follows, of length L+1 */ - if(ctrl < (1 << 5)) { - - ctrl++; - - if(fx_expect_false(op + ctrl > out_end)){ - --ip; /* Rewind to control byte */ - goto guess; - } - if(fx_expect_false(ip + ctrl > in_end)) return LZFX_ECORRUPT; - - do - *op++ = *ip++; - while(--ctrl); - - /* Format #1 [LLLooooo oooooooo]: backref of length L+1+2 - ^^^^^ ^^^^^^^^ - A B - #2 [111ooooo LLLLLLLL oooooooo] backref of length L+7+2 - ^^^^^ ^^^^^^^^ - A B - In both cases the location of the backref is computed from the - remaining part of the data as follows: - - location = op - A*256 - B - 1 - */ - } else { - - unsigned int len = (ctrl >> 5); - u8 *ref = op - ((ctrl & 0x1f) << 8) -1; - - if(len==7) len += *ip++; /* i.e. format #2 */ - - len += 2; /* len is now #octets */ - - if(fx_expect_false(op + len > out_end)){ - ip -= (len >= 9) ? 2 : 1; /* Rewind to control byte */ - goto guess; - } - if(fx_expect_false(ip >= in_end)) return LZFX_ECORRUPT; - - ref -= *ip++; - - if(fx_expect_false(ref < (u8*)obuf)) return LZFX_ECORRUPT; - - do - *op++ = *ref++; - while (--len); - } - - } while (ip < in_end); - - *olen = op - (u8 *)obuf; - - return 0; - -guess: - rc = lzfx_getsize(ip, ilen - (ip-(u8*)ibuf), &remain_len); - if(rc>=0) *olen = remain_len + (op - (u8*)obuf); - return rc; -} - -/* Guess len. No parameters may be NULL; this is not checked. */ -static -int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen){ - - u8 const *ip = (const u8 *)ibuf; - u8 const *const in_end = ip + ilen; - int tot_len = 0; - - while (ip < in_end) { - - unsigned int ctrl = *ip++; - - if(ctrl < (1 << 5)) { - - ctrl++; - - if(ip + ctrl > in_end) - return LZFX_ECORRUPT; - - tot_len += ctrl; - ip += ctrl; - - } else { - - unsigned int len = (ctrl >> 5); - - if(len==7){ /* i.e. format #2 */ - len += *ip++; - } - - len += 2; /* len is now #octets */ - - if(ip >= in_end) return LZFX_ECORRUPT; - - ip++; /* skip the ref byte */ - - tot_len += len; - - } - - } - - *olen = tot_len; - - return 0; -} - - - - diff --git a/Modem/compression/lzfx.h b/Modem/compression/lzfx.h deleted file mode 100644 index 13c9e8f..0000000 --- a/Modem/compression/lzfx.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2009 Andrew Collette - * http://lzfx.googlecode.com - * - * Implements an LZF-compatible compressor/decompressor based on the liblzf - * codebase written by Marc Lehmann. This code is released under the BSD - * license. License and original copyright statement follow. - * - * - * Copyright (c) 2000-2008 Marc Alexander Lehmann - * - * Redistribution and use in source and binary forms, with or without modifica- - * tion, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- - * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- - * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- - * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZFX_H -#define LZFX_H - -/* Documented behavior, including function signatures and error codes, - is guaranteed to remain unchanged for releases with the same major - version number. Releases of the same major version are also able - to read each other's output, although the output itself is not - guaranteed to be byte-for-byte identical. -*/ -#define LZFX_VERSION_MAJOR 0 -#define LZFX_VERSION_MINOR 1 -#define LZFX_VERSION_STRING "0.1" - -/* Hashtable size (2**LZFX_HLOG entries) */ -#ifndef LZFX_HLOG -# define LZFX_HLOG 8 -#endif - -/* Predefined errors. */ -#define LZFX_ESIZE -1 /* Output buffer too small */ -#define LZFX_ECORRUPT -2 /* Invalid data for decompression */ -#define LZFX_EARGS -3 /* Arguments invalid (NULL) */ - -/* Buffer-to buffer compression. - - Supply pre-allocated input and output buffers via ibuf and obuf, and - their size in bytes via ilen and olen. Buffers may not overlap. - - On success, the function returns a non-negative value and the argument - olen contains the compressed size in bytes. On failure, a negative - value is returned and olen is not modified. -*/ -int lzfx_compress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen); - -/* Buffer-to-buffer decompression. - - Supply pre-allocated input and output buffers via ibuf and obuf, and - their size in bytes via ilen and olen. Buffers may not overlap. - - On success, the function returns a non-negative value and the argument - olen contains the uncompressed size in bytes. On failure, a negative - value is returned. - - If the failure code is LZFX_ESIZE, olen contains the minimum buffer size - required to hold the decompressed data. Otherwise, olen is not modified. - - Supplying a zero *olen is a valid and supported strategy to determine the - required buffer size. This does not require decompression of the entire - stream and is consequently very fast. Argument obuf may be NULL in - this case only. -*/ -int lzfx_decompress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen); - - -#endif diff --git a/Modem/main.c b/Modem/main.c index a933215..5e349ac 100644 --- a/Modem/main.c +++ b/Modem/main.c @@ -15,6 +15,8 @@ #include "afsk.h" // Header for AFSK modem #include "protocol/mp1.h" // Header for MP.1 protocol +#include "compression/heatshrink_encoder.h" +#include "compression/heatshrink_decoder.h" ////////////////////////////////////////////////////// // A few definitions // @@ -39,6 +41,114 @@ static int serialLen = 0; // Counter for counting length of data from seri static bool sertx = false; // Flag signifying whether it's time to send data // Received on the serial port. +static uint8_t compressedData[MP1_MAX_FRAME_LENGTH]; +static uint8_t decompressedData[MP1_MAX_FRAME_LENGTH]; + + +static int freeRam () { + extern int __heap_start, *__brkval; + int v; + return (int) &v - (__brkval == 0 ? (int) &__heap_start : (int) __brkval); +} + +static size_t compress(uint8_t *input, size_t length) { + heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4); + if (hse == NULL) { + kprintf("Could not allocate encoder"); + return 0; + } + + size_t written = 0; + size_t sunk = 0; + heatshrink_encoder_sink(hse, input, length, &sunk); + int status = heatshrink_encoder_finish(hse); + + if (sunk < length) { + kprintf("Not all data was sunk into encoder\n"); + heatshrink_encoder_free(hse); + return 0; + } else { + kprintf("Bytes sunk into HSE: %d\n", length); + if (status == HSER_FINISH_MORE) { + heatshrink_encoder_poll(hse, compressedData, MP1_MAX_FRAME_LENGTH, &written); + kprintf("Bytes written into buffer: %d\n", written); + } else { + kprintf("All input data was sunk, but encoder doesn't have any data for us."); + } + } + + heatshrink_encoder_free(hse); + return written; +} + +static size_t decompress(uint8_t *input, size_t length) { + heatshrink_decoder *hsd = heatshrink_decoder_alloc(MP1_MAX_FRAME_LENGTH, 8, 4); + if (hsd == NULL) { + kprintf("Could not allocate decoder"); + return 0; + } + + kprintf("\nDecoder allocated. Free RAM: %d bytes\n", freeRam()); + + size_t written = 0; + size_t sunk = 0; + heatshrink_decoder_sink(hsd, input, length, &sunk); + int status = heatshrink_decoder_finish(hsd); + + if (sunk < length) { + kprintf("Not all data was sunk into decoder\n"); + heatshrink_decoder_free(hsd); + return 0; + } else { + kprintf("Bytes sunk into HSD: %d\n", length); + if (status == HSER_FINISH_MORE) { + heatshrink_decoder_poll(hsd, decompressedData, MP1_MAX_FRAME_LENGTH, &written); + kprintf("Bytes written into decompression buffer: %d\n", written); + } else { + kprintf("All input data was sunk, but the decoder doesn't have any data for us."); + } + } + + heatshrink_decoder_free(hsd); + return written; +} + +static void hseTest() { + kprintf("\nFree RAM: %d bytes\n", freeRam()); + size_t compressed_size = compress(serialBuffer, serialLen); + size_t decompressed_size = decompress(compressedData, compressed_size); + kprintf("\n-------------------\nInput size: %d\nCompressed size: %d\nDecompressed size: %d\n", serialLen, compressed_size, decompressed_size); + + // heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4); + // kprintf("\nFree RAM: %d bytes\n", freeRam()); + + // size_t out_sz = 50; + // uint8_t out_buf[out_sz]; + // size_t written = 0; + // kprintf("\nFree RAM: %d bytes\n", freeRam()); + + // size_t length = serialLen; + + // heatshrink_encoder_sink(hse, serialBuffer, serialLen, &length); + + + + // int returnv = heatshrink_encoder_finish(hse); + // kprintf("Encoder finish returned: %d\n", returnv); + + // if (length < serialLen) { + // kprintf("Not all data was sunk into encoder\n"); + // } else { + // // All data delivered + // kprintf("Bytes sunk into HSE: %d\n", length); + + // heatshrink_encoder_poll(hse, out_buf, out_sz, &written); + + // kprintf("2: Bytes written into buffer: %d\n", written); + // } + + // heatshrink_encoder_free(hse); +} ////////////////////////////////////////////////////// // And here comes the actual program :) // @@ -52,12 +162,6 @@ static void mp1Callback(struct MP1Packet *packet) { //kprintf("%.*s\n", packet->dataLength, packet->data); } -// static int freeRam () { -// extern int __heap_start, *__brkval; -// int v; -// return (int) &v - (__brkval == 0 ? (int) &__heap_start : (int) __brkval); -// } - // Simple initialization function. static void init(void) { @@ -125,7 +229,9 @@ int main(void) if (sertx) { // If we should, pass the buffer to the protocol's // send function. - mp1Send(&mp1, serialBuffer, serialLen); + + hseTest(); + // mp1Send(&mp1, serialBuffer, serialLen); // Reset the transmission flag and length counter sertx = false; diff --git a/Modem/protocol/mp1.h b/Modem/protocol/mp1.h index a41bb95..2f842b0 100644 --- a/Modem/protocol/mp1.h +++ b/Modem/protocol/mp1.h @@ -4,8 +4,6 @@ #include #include -#include "compression/lzfx.h" - // Frame sizing & checksum #define MP1_MIN_FRAME_LENGTH 3 #define MP1_MAX_FRAME_LENGTH 200 diff --git a/buildrev.h b/buildrev.h index ef1cde4..900630e 100644 --- a/buildrev.h +++ b/buildrev.h @@ -1,2 +1,2 @@ -#define VERS_BUILD 930 +#define VERS_BUILD 994 #define VERS_HOST "vixen"