Mirrors
/
OpenModem
mirror of https://github.com/markqvist/OpenModem.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Successfull compression test with heatshrink

This commit is contained in:
Mark Qvist 2014-04-14 17:38:35 +02:00
parent 434cf6f2e6
commit 99951676c0
14 changed files with 1386 additions and 1105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Modem/Modem_user.mk
View File

@ -13,6 +13,8 @@ Modem_USER_CSRC = \
$(Modem_HW_PATH)/hardware.c \
$(Modem_HW_PATH)/afsk.c \
$(Modem_HW_PATH)/protocol/mp1.c \
$(Modem_HW_PATH)/compression/heatshrink_decoder.c \
$(Modem_HW_PATH)/compression/heatshrink_encoder.c \
#
# Files included by the user.

543
Modem/compression/fastlz.c
View File

@ -1,543 +0,0 @@
/*
FastLZ - lightning-fast lossless compression library
Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2006 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2005 Ariya Hidayat (ariya@kde.org)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#if !defined(FASTLZ__COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR)
/*
* Always check for bound when decompressing.
* Generally it is best to leave it defined.
*/
#define FASTLZ_SAFE
/*
* Give hints to the compiler for branch prediction optimization.
*/
#if defined(__GNUC__) && (__GNUC__ > 2)
#define FASTLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1))
#define FASTLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0))
#else
#define FASTLZ_EXPECT_CONDITIONAL(c) (c)
#define FASTLZ_UNEXPECT_CONDITIONAL(c) (c)
#endif
/*
* Use inlined functions for supported systems.
*/
#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C)
#define FASTLZ_INLINE inline
#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__)
#define FASTLZ_INLINE __inline
#else
#define FASTLZ_INLINE
#endif
/*
* Prevent accessing more than 8-bit at once, except on x86 architectures.
*/
#if !defined(FASTLZ_STRICT_ALIGN)
#define FASTLZ_STRICT_ALIGN
#if defined(__i386__) || defined(__386) /* GNU C, Sun Studio */
#undef FASTLZ_STRICT_ALIGN
#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */
#undef FASTLZ_STRICT_ALIGN
#elif defined(_M_IX86) /* Intel, MSVC */
#undef FASTLZ_STRICT_ALIGN
#elif defined(__386)
#undef FASTLZ_STRICT_ALIGN
#elif defined(_X86_) /* MinGW */
#undef FASTLZ_STRICT_ALIGN
#elif defined(__I86__) /* Digital Mars */
#undef FASTLZ_STRICT_ALIGN
#endif
#endif
/*
* FIXME: use preprocessor magic to set this on different platforms!
*/
typedef unsigned char flzuint8;
typedef unsigned short flzuint16;
typedef unsigned int flzuint32;
/* prototypes */
int fastlz_compress(const void* input, int length, void* output);
int fastlz_compress_level(int level, const void* input, int length, void* output);
int fastlz_decompress(const void* input, int length, void* output, int maxout);
#define MAX_COPY 32
#define MAX_LEN 264 /* 256 + 8 */
#define MAX_DISTANCE 8192
#if !defined(FASTLZ_STRICT_ALIGN)
#define FASTLZ_READU16(p) *((const flzuint16*)(p))
#else
#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8)
#endif
#define HASH_LOG 13
#define HASH_SIZE (1<< HASH_LOG)
#define HASH_MASK (HASH_SIZE-1)
#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; }
#undef FASTLZ_LEVEL
#define FASTLZ_LEVEL 1
#undef FASTLZ_COMPRESSOR
#undef FASTLZ_DECOMPRESSOR
#define FASTLZ_COMPRESSOR fastlz1_compress
#define FASTLZ_DECOMPRESSOR fastlz1_decompress
static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output);
static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout);
#include "fastlz.c"
#undef FASTLZ_LEVEL
#define FASTLZ_LEVEL 2
#undef MAX_DISTANCE
#define MAX_DISTANCE 8191
#define MAX_FARDISTANCE (65535+MAX_DISTANCE-1)
#undef FASTLZ_COMPRESSOR
#undef FASTLZ_DECOMPRESSOR
#define FASTLZ_COMPRESSOR fastlz2_compress
#define FASTLZ_DECOMPRESSOR fastlz2_decompress
static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output);
static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout);
#include "fastlz.c"
int fastlz_compress(const void* input, int length, void* output)
{
return fastlz1_compress(input, length, output);
// Fastlz2 doesn't really matter for us
// because of small block sizes
// return fastlz2_compress(input, length, output);
}
int fastlz_decompress(const void* input, int length, void* output, int maxout)
{
/* magic identifier for compression level */
int level = ((*(const flzuint8*)input) >> 5) + 1;
if(level == 1)
return fastlz1_decompress(input, length, output, maxout);
if(level == 2)
return fastlz2_decompress(input, length, output, maxout);
/* unknown level, trigger error */
return 0;
}
int fastlz_compress_level(int level, const void* input, int length, void* output)
{
if(level == 1)
return fastlz1_compress(input, length, output);
if(level == 2)
return fastlz2_compress(input, length, output);
return 0;
}
#else /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */
static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output)
{
const flzuint8* ip = (const flzuint8*) input;
const flzuint8* ip_bound = ip + length - 2;
const flzuint8* ip_limit = ip + length - 12;
flzuint8* op = (flzuint8*) output;
const flzuint8* htab[HASH_SIZE];
const flzuint8** hslot;
flzuint32 hval;
flzuint32 copy;
/* sanity check */
if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4))
{
if(length)
{
/* create literal copy only */
*op++ = length-1;
ip_bound++;
while(ip <= ip_bound)
*op++ = *ip++;
return length+1;
}
else
return 0;
}
/* initializes hash table */
for (hslot = htab; hslot < htab + HASH_SIZE; hslot++)
*hslot = ip;
/* we start with literal copy */
copy = 2;
*op++ = MAX_COPY-1;
*op++ = *ip++;
*op++ = *ip++;
/* main loop */
while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit))
{
const flzuint8* ref;
flzuint32 distance;
/* minimum match length */
flzuint32 len = 3;
/* comparison starting-point */
const flzuint8* anchor = ip;
/* check for a run */
#if FASTLZ_LEVEL==2
if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1))
{
distance = 1;
ip += 3;
ref = anchor - 1 + 3;
goto match;
}
#endif
/* find potential match */
HASH_FUNCTION(hval,ip);
hslot = htab + hval;
ref = htab[hval];
/* calculate distance to the match */
distance = anchor - ref;
/* update hash table */
*hslot = anchor;
/* is this a match? check the first 3 bytes */
if(distance==0 || (distance >= MAX_DISTANCE) || *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++) goto literal;
#if FASTLZ_LEVEL==2
/* far, needs at least 5-byte match */
if(distance >= MAX_DISTANCE)
{
if(*ip++ != *ref++ || *ip++!= *ref++)
goto literal;
len += 2;
}
match:
#endif
/* last matched byte */
ip = anchor + len;
/* distance is biased */
distance--;
if(!distance)
{
/* zero distance means a run */
flzuint8 x = ip[-1];
while(ip < ip_bound)
if(*ref++ != x) break; else ip++;
}
else
for(;;)
{
/* safe because the outer check against ip limit */
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
while(ip < ip_bound)
if(*ref++ != *ip++) break;
break;
}
/* if we have copied something, adjust the copy count */
if(copy)
/* copy is biased, '0' means 1 byte copy */
*(op-copy-1) = copy-1;
else
/* back, to overwrite the copy count */
op--;
/* reset literal counter */
copy = 0;
/* length is biased, '1' means a match of 3 bytes */
ip -= 3;
len = ip - anchor;
/* encode the match */
#if FASTLZ_LEVEL==2
if(distance < MAX_DISTANCE)
{
if(len < 7)
{
*op++ = (len << 5) + (distance >> 8);
*op++ = (distance & 255);
}
else
{
*op++ = (7 << 5) + (distance >> 8);
for(len-=7; len >= 255; len-= 255)
*op++ = 255;
*op++ = len;
*op++ = (distance & 255);
}
}
else
{
/* far away, but not yet in the another galaxy... */
if(len < 7)
{
distance -= MAX_DISTANCE;
*op++ = (len << 5) + 31;
*op++ = 255;
*op++ = distance >> 8;
*op++ = distance & 255;
}
else
{
distance -= MAX_DISTANCE;
*op++ = (7 << 5) + 31;
for(len-=7; len >= 255; len-= 255)
*op++ = 255;
*op++ = len;
*op++ = 255;
*op++ = distance >> 8;
*op++ = distance & 255;
}
}
#else
if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2))
while(len > MAX_LEN-2)
{
*op++ = (7 << 5) + (distance >> 8);
*op++ = MAX_LEN - 2 - 7 -2;
*op++ = (distance & 255);
len -= MAX_LEN-2;
}
if(len < 7)
{
*op++ = (len << 5) + (distance >> 8);
*op++ = (distance & 255);
}
else
{
*op++ = (7 << 5) + (distance >> 8);
*op++ = len - 7;
*op++ = (distance & 255);
}
#endif
/* update the hash at match boundary */
HASH_FUNCTION(hval,ip);
htab[hval] = ip++;
HASH_FUNCTION(hval,ip);
htab[hval] = ip++;
/* assuming literal copy */
*op++ = MAX_COPY-1;
continue;
literal:
*op++ = *anchor++;
ip = anchor;
copy++;
if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY))
{
copy = 0;
*op++ = MAX_COPY-1;
}
}
/* left-over as literal copy */
ip_bound++;
while(ip <= ip_bound)
{
*op++ = *ip++;
copy++;
if(copy == MAX_COPY)
{
copy = 0;
*op++ = MAX_COPY-1;
}
}
/* if we have copied something, adjust the copy length */
if(copy)
*(op-copy-1) = copy-1;
else
op--;
#if FASTLZ_LEVEL==2
/* marker for fastlz2 */
*(flzuint8*)output |= (1 << 5);
#endif
return op - (flzuint8*)output;
}
static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout)
{
const flzuint8* ip = (const flzuint8*) input;
const flzuint8* ip_limit = ip + length;
flzuint8* op = (flzuint8*) output;
flzuint8* op_limit = op + maxout;
flzuint32 ctrl = (*ip++) & 31;
int loop = 1;
do
{
const flzuint8* ref = op;
flzuint32 len = ctrl >> 5;
flzuint32 ofs = (ctrl & 31) << 8;
if(ctrl >= 32)
{
#if FASTLZ_LEVEL==2
flzuint8 code;
#endif
len--;
ref -= ofs;
if (len == 7-1)
#if FASTLZ_LEVEL==1
len += *ip++;
ref -= *ip++;
#else
do
{
code = *ip++;
len += code;
} while (code==255);
code = *ip++;
ref -= code;
/* match from 16-bit distance */
if(FASTLZ_UNEXPECT_CONDITIONAL(code==255))
if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8)))
{
ofs = (*ip++) << 8;
ofs += *ip++;
ref = op - ofs - MAX_DISTANCE;
}
#endif
#ifdef FASTLZ_SAFE
if (FASTLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit))
return 0;
if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output))
return 0;
#endif
if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit))
ctrl = *ip++;
else
loop = 0;
if(ref == op)
{
/* optimize copy for a run */
flzuint8 b = ref[-1];
*op++ = b;
*op++ = b;
*op++ = b;
for(; len; --len)
*op++ = b;
}
else
{
#if !defined(FASTLZ_STRICT_ALIGN)
const flzuint16* p;
flzuint16* q;
#endif
/* copy from reference */
ref--;
*op++ = *ref++;
*op++ = *ref++;
*op++ = *ref++;
#if !defined(FASTLZ_STRICT_ALIGN)
/* copy a byte, so that now it's word aligned */
if(len & 1)
{
*op++ = *ref++;
len--;
}
/* copy 16-bit at once */
q = (flzuint16*) op;
op += len;
p = (const flzuint16*) ref;
for(len>>=1; len > 4; len-=4)
{
*q++ = *p++;
*q++ = *p++;
*q++ = *p++;
*q++ = *p++;
}
for(; len; --len)
*q++ = *p++;
#else
for(; len; --len)
*op++ = *ref++;
#endif
}
}
else
{
ctrl++;
#ifdef FASTLZ_SAFE
if (FASTLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit))
return 0;
if (FASTLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit))
return 0;
#endif
*op++ = *ip++;
for(--ctrl; ctrl; ctrl--)
*op++ = *ip++;
loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit);
if(loop)
ctrl = *ip++;
}
}
while(FASTLZ_EXPECT_CONDITIONAL(loop));
return op - (flzuint8*)output;
}
#endif /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */

100
Modem/compression/fastlz.h
View File

@ -1,100 +0,0 @@
/*
FastLZ - lightning-fast lossless compression library
Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2006 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2005 Ariya Hidayat (ariya@kde.org)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef FASTLZ_H
#define FASTLZ_H
#define FASTLZ_VERSION 0x000100
#define FASTLZ_VERSION_MAJOR 0
#define FASTLZ_VERSION_MINOR 0
#define FASTLZ_VERSION_REVISION 0
#define FASTLZ_VERSION_STRING "0.1.0"
#if defined (__cplusplus)
extern "C" {
#endif
/**
Compress a block of data in the input buffer and returns the size of
compressed block. The size of input buffer is specified by length. The
minimum input buffer size is 16.
The output buffer must be at least 5% larger than the input buffer
and can not be smaller than 66 bytes.
If the input is not compressible, the return value might be larger than
length (input buffer size).
The input buffer and the output buffer can not overlap.
*/
int fastlz_compress(const void* input, int length, void* output);
/**
Decompress a block of compressed data and returns the size of the
decompressed block. If error occurs, e.g. the compressed data is
corrupted or the output buffer is not large enough, then 0 (zero)
will be returned instead.
The input buffer and the output buffer can not overlap.
Decompression is memory safe and guaranteed not to write the output buffer
more than what is specified in maxout.
*/
int fastlz_decompress(const void* input, int length, void* output, int maxout);
/**
Compress a block of data in the input buffer and returns the size of
compressed block. The size of input buffer is specified by length. The
minimum input buffer size is 16.
The output buffer must be at least 5% larger than the input buffer
and can not be smaller than 66 bytes.
If the input is not compressible, the return value might be larger than
length (input buffer size).
The input buffer and the output buffer can not overlap.
Compression level can be specified in parameter level. At the moment,
only level 1 and level 2 are supported.
Level 1 is the fastest compression and generally useful for short data.
Level 2 is slightly slower but it gives better compression ratio.
Note that the compressed data, regardless of the level, can always be
decompressed using the function fastlz_decompress above.
*/
int fastlz_compress_level(int level, const void* input, int length, void* output);
#if defined (__cplusplus)
}
#endif
#endif /* FASTLZ_H */

19
Modem/compression/heatshrink_common.h Normal file
View File

@ -0,0 +1,19 @@
#ifndef HEATSHRINK_H
#define HEATSHRINK_H
#define HEATSHRINK_AUTHOR "Scott Vokes <scott.vokes@atomicobject.com>"
/* Version 0.3.0 */
#define HEATSHRINK_VERSION_MAJOR 0
#define HEATSHRINK_VERSION_MINOR 3
#define HEATSHRINK_VERSION_PATCH 0
#define HEATSHRINK_MIN_WINDOW_BITS 4
#define HEATSHRINK_MAX_WINDOW_BITS 15
#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2
#define HEATSHRINK_LITERAL_MARKER 0x01
#define HEATSHRINK_BACKREF_MARKER 0x00
#endif

24
Modem/compression/heatshrink_config.h Normal file
View File

@ -0,0 +1,24 @@
#ifndef HEATSHRINK_CONFIG_H
#define HEATSHRINK_CONFIG_H
/* Should functionality assuming dynamic allocation be used? */
#define HEATSHRINK_DYNAMIC_ALLOC 1
#if HEATSHRINK_DYNAMIC_ALLOC
/* Optional replacement of malloc/free */
#define HEATSHRINK_MALLOC(SZ) malloc(SZ)
#define HEATSHRINK_FREE(P, SZ) free(P)
#else
/* Required parameters for static configuration */
#define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 200
#define HEATSHRINK_STATIC_WINDOW_BITS 8
#define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4
#endif
/* Turn on logging for debugging. */
#define HEATSHRINK_DEBUGGING_LOGS 0
/* Use indexing for faster compression. (This requires additional space.) */
#define HEATSHRINK_USE_INDEX 0
#endif

376
Modem/compression/heatshrink_decoder.c Normal file
View File

@ -0,0 +1,376 @@
#include <stdlib.h>
#include <string.h>
#include "heatshrink_decoder.h"
/* States for the polling state machine. */
typedef enum {
HSDS_EMPTY, /* no input to process */
HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */
HSDS_YIELD_LITERAL, /* ready to yield literal byte */
HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */
HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */
HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */
HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */
HSDS_YIELD_BACKREF, /* ready to yield back-reference */
HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */
} HSD_state;
#if HEATSHRINK_DEBUGGING_LOGS
#include <stdio.h>
#include <ctype.h>
#include <assert.h>
#define LOG(...) fprintf(stderr, __VA_ARGS__)
#define ASSERT(X) assert(X)
static const char *state_names[] = {
"empty",
"input_available",
"yield_literal",
"backref_index",
"backref_count",
"yield_backref",
"check_for_more_input",
};
#else
#define LOG(...) /* no-op */
#define ASSERT(X) /* no-op */
#endif
typedef struct {
uint8_t *buf; /* output buffer */
size_t buf_size; /* buffer size */
size_t *output_size; /* bytes pushed to buffer, so far */
} output_info;
#define NO_BITS ((uint32_t)-1)
/* Forward references. */
static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count);
static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte);
#if HEATSHRINK_DYNAMIC_ALLOC
heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size,
uint8_t window_sz2,
uint8_t lookahead_sz2) {
if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) ||
(window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) ||
(input_buffer_size == 0) ||
(lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) ||
(lookahead_sz2 > window_sz2)) {
return NULL;
}
size_t buffers_sz = (1 << window_sz2) + input_buffer_size;
size_t sz = sizeof(heatshrink_decoder) + buffers_sz;
heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz);
if (hsd == NULL) { return NULL; }
hsd->input_buffer_size = input_buffer_size;
hsd->window_sz2 = window_sz2;
hsd->lookahead_sz2 = lookahead_sz2;
heatshrink_decoder_reset(hsd);
LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n",
sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size);
return hsd;
}
void heatshrink_decoder_free(heatshrink_decoder *hsd) {
size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size;
size_t sz = sizeof(heatshrink_decoder) + buffers_sz;
HEATSHRINK_FREE(hsd, sz);
(void)sz; /* may not be used by free */
}
#endif
void heatshrink_decoder_reset(heatshrink_decoder *hsd) {
size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd);
size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd);
memset(hsd->buffers, 0, buf_sz + input_sz);
hsd->state = HSDS_EMPTY;
hsd->input_size = 0;
hsd->input_index = 0;
hsd->bit_index = 0x00;
hsd->current_byte = 0x00;
hsd->output_count = 0;
hsd->output_index = 0;
hsd->head_index = 0;
hsd->bit_accumulator = 0x00000000;
}
/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */
HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd,
uint8_t *in_buf, size_t size, size_t *input_size) {
if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) {
return HSDR_SINK_ERROR_NULL;
}
size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size;
if (rem == 0) {
*input_size = 0;
return HSDR_SINK_FULL;
}
size = rem < size ? rem : size;
LOG("-- sinking %zd bytes\n", size);
/* copy into input buffer (at head of buffers) */
memcpy(&hsd->buffers[hsd->input_size], in_buf, size);
hsd->input_size += size;
if (hsd->state == HSDS_EMPTY) {
hsd->state = HSDS_INPUT_AVAILABLE;
hsd->input_index = 0;
}
*input_size = size;
return HSDR_SINK_OK;
}
/*****************
* Decompression *
*****************/
#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD))
#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD))
// States
static HSD_state st_input_available(heatshrink_decoder *hsd);
static HSD_state st_yield_literal(heatshrink_decoder *hsd,
output_info *oi);
static HSD_state st_backref_index_msb(heatshrink_decoder *hsd);
static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd);
static HSD_state st_backref_count_msb(heatshrink_decoder *hsd);
static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd);
static HSD_state st_yield_backref(heatshrink_decoder *hsd,
output_info *oi);
static HSD_state st_check_for_input(heatshrink_decoder *hsd);
HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size) {
if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) {
return HSDR_POLL_ERROR_NULL;
}
*output_size = 0;
output_info oi;
oi.buf = out_buf;
oi.buf_size = out_buf_size;
oi.output_size = output_size;
while (1) {
LOG("-- poll, state is %d (%s), input_size %d\n",
hsd->state, state_names[hsd->state], hsd->input_size);
uint8_t in_state = hsd->state;
switch (in_state) {
case HSDS_EMPTY:
return HSDR_POLL_EMPTY;
case HSDS_INPUT_AVAILABLE:
hsd->state = st_input_available(hsd);
break;
case HSDS_YIELD_LITERAL:
hsd->state = st_yield_literal(hsd, &oi);
break;
case HSDS_BACKREF_INDEX_MSB:
hsd->state = st_backref_index_msb(hsd);
break;
case HSDS_BACKREF_INDEX_LSB:
hsd->state = st_backref_index_lsb(hsd);
break;
case HSDS_BACKREF_COUNT_MSB:
hsd->state = st_backref_count_msb(hsd);
break;
case HSDS_BACKREF_COUNT_LSB:
hsd->state = st_backref_count_lsb(hsd);
break;
case HSDS_YIELD_BACKREF:
hsd->state = st_yield_backref(hsd, &oi);
break;
case HSDS_CHECK_FOR_MORE_INPUT:
hsd->state = st_check_for_input(hsd);
break;
default:
return HSDR_POLL_ERROR_UNKNOWN;
}
/* If the current state cannot advance, check if input or output
* buffer are exhausted. */
if (hsd->state == in_state) {
if (*output_size == out_buf_size) { return HSDR_POLL_MORE; }
return HSDR_POLL_EMPTY;
}
}
}
static HSD_state st_input_available(heatshrink_decoder *hsd) {
uint32_t bits = get_bits(hsd, 1); // get tag bit
if (bits) {
return HSDS_YIELD_LITERAL;
} else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) {
return HSDS_BACKREF_INDEX_MSB;
} else {
hsd->output_index = 0;
return HSDS_BACKREF_INDEX_LSB;
}
}
static HSD_state st_yield_literal(heatshrink_decoder *hsd,
output_info *oi) {
/* Emit a repeated section from the window buffer, and add it (again)
* to the window buffer. (Note that the repetition can include
* itself.)*/
if (*oi->output_size < oi->buf_size) {
uint32_t byte = get_bits(hsd, 8);
if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */
uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)];
uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1;
uint8_t c = byte & 0xFF;
LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.');
buf[hsd->head_index++ & mask] = c;
push_byte(hsd, oi, c);
return HSDS_CHECK_FOR_MORE_INPUT;
} else {
return HSDS_YIELD_LITERAL;
}
}
static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) {
uint8_t bit_ct = BACKREF_INDEX_BITS(hsd);
ASSERT(bit_ct > 8);
uint32_t bits = get_bits(hsd, bit_ct - 8);
LOG("-- backref index (msb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; }
hsd->output_index = bits << 8;
return HSDS_BACKREF_INDEX_LSB;
}
static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) {
uint8_t bit_ct = BACKREF_INDEX_BITS(hsd);
uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8);
LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; }
hsd->output_index |= bits;
hsd->output_index++;
uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd);
hsd->output_count = 0;
return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB;
}
static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) {
uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd);
ASSERT(br_bit_ct > 8);
uint32_t bits = get_bits(hsd, br_bit_ct - 8);
LOG("-- backref count (msb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; }
hsd->output_count = bits << 8;
return HSDS_BACKREF_COUNT_LSB;
}
static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) {
uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd);
uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8);
LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; }
hsd->output_count |= bits;
hsd->output_count++;
return HSDS_YIELD_BACKREF;
}
static HSD_state st_yield_backref(heatshrink_decoder *hsd,
output_info *oi) {
size_t count = oi->buf_size - *oi->output_size;
if (count > 0) {
if (hsd->output_count < count) count = hsd->output_count;
uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)];
uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1;
uint16_t neg_offset = hsd->output_index;
LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset);
ASSERT(neg_offset < mask + 1);
ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd));
for (size_t i=0; i<count; i++) {
uint8_t c = buf[(hsd->head_index - neg_offset) & mask];
push_byte(hsd, oi, c);
buf[hsd->head_index & mask] = c;
hsd->head_index++;
LOG(" -- ++ 0x%02x\n", c);
}
hsd->output_count -= count;
if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; }
}
return HSDS_YIELD_BACKREF;
}
static HSD_state st_check_for_input(heatshrink_decoder *hsd) {
return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE;
}
/* Get the next COUNT bits from the input buffer, saving incremental progress.
* Returns NO_BITS on end of input, or if more than 31 bits are requested. */
static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) {
if (count > 31) { return NO_BITS; }
LOG("-- popping %u bit(s)\n", count);
/* If we aren't able to get COUNT bits, suspend immediately, because we
* don't track how many bits of COUNT we've accumulated before suspend. */
if (hsd->input_size == 0) {
if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; }
}
for (int i = 0; i < count; i++) {
if (hsd->bit_index == 0x00) {
if (hsd->input_size == 0) {
LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n",
hsd->bit_accumulator, hsd->bit_accumulator);
return NO_BITS;
}
hsd->current_byte = hsd->buffers[hsd->input_index++];
LOG(" -- pulled byte 0x%02x\n", hsd->current_byte);
if (hsd->input_index == hsd->input_size) {
hsd->input_index = 0; /* input is exhausted */
hsd->input_size = 0;
}
hsd->bit_index = 0x80;
}
hsd->bit_accumulator <<= 1;
if (hsd->current_byte & hsd->bit_index) {
hsd->bit_accumulator |= 0x01;
if (0) {
LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n",
hsd->bit_accumulator, hsd->bit_index);
}
} else {
if (0) {
LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n",
hsd->bit_accumulator, hsd->bit_index);
}
}
hsd->bit_index >>= 1;
}
uint32_t res = 0;
res = hsd->bit_accumulator;
hsd->bit_accumulator = 0x00000000;
if (count > 1) { LOG(" -- accumulated %08x\n", res); }
return res;
}
HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) {
if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; }
switch (hsd->state) {
case HSDS_EMPTY:
return HSDR_FINISH_DONE;
/* If we want to finish with no input, but are in these states, it's
* because the 0-bit padding to the last byte looks like a backref
* marker bit followed by all 0s for index and count bits. */
case HSDS_BACKREF_INDEX_LSB:
case HSDS_BACKREF_INDEX_MSB:
case HSDS_BACKREF_COUNT_LSB:
case HSDS_BACKREF_COUNT_MSB:
return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE;
/* fall through */
default:
return HSDR_FINISH_MORE;
}
}
static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) {
LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.');
oi->buf[(*oi->output_size)++] = byte;
(void)hsd;
}

101
Modem/compression/heatshrink_decoder.h Normal file
View File

@ -0,0 +1,101 @@
#ifndef HEATSHRINK_DECODER_H
#define HEATSHRINK_DECODER_H
#include <stdint.h>
#include <stddef.h>
#include "heatshrink_common.h"
#include "heatshrink_config.h"
typedef enum {
HSDR_SINK_OK, /* data sunk, ready to poll */
HSDR_SINK_FULL, /* out of space in internal buffer */
HSDR_SINK_ERROR_NULL=-1, /* NULL argument */
} HSD_sink_res;
typedef enum {
HSDR_POLL_EMPTY, /* input exhausted */
HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */
HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */
HSDR_POLL_ERROR_UNKNOWN=-2,
} HSD_poll_res;
typedef enum {
HSDR_FINISH_DONE, /* output is done */
HSDR_FINISH_MORE, /* more output remains */
HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */
} HSD_finish_res;
#if HEATSHRINK_DYNAMIC_ALLOC
#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \
((BUF)->input_buffer_size)
#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \
((BUF)->window_sz2)
#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \
((BUF)->lookahead_sz2)
#else
#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \
HEATSHRINK_STATIC_INPUT_BUFFER_SIZE
#define HEATSHRINK_DECODER_WINDOW_BITS(_) \
(HEATSHRINK_STATIC_WINDOW_BITS)
#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \
(HEATSHRINK_STATIC_LOOKAHEAD_BITS)
#endif
typedef struct {
uint16_t input_size; /* bytes in input buffer */
uint16_t input_index; /* offset to next unprocessed input byte */
uint16_t output_count; /* how many bytes to output */
uint16_t output_index; /* index for bytes to output */
uint16_t head_index; /* head of window buffer */
uint16_t bit_accumulator;
uint8_t state; /* current state machine node */
uint8_t current_byte; /* current byte of input */
uint8_t bit_index; /* current bit index */
#if HEATSHRINK_DYNAMIC_ALLOC
/* Fields that are only used if dynamically allocated. */
uint8_t window_sz2; /* window buffer bits */
uint8_t lookahead_sz2; /* lookahead bits */
uint16_t input_buffer_size; /* input buffer size */
/* Input buffer, then expansion window buffer */
uint8_t buffers[];
#else
/* Input buffer, then expansion window buffer */
uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_))
+ HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)];
#endif
} heatshrink_decoder;
#if HEATSHRINK_DYNAMIC_ALLOC
/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes,
* an expansion buffer size of 2^WINDOW_SZ2, and a lookahead
* size of 2^lookahead_sz2. (The window buffer and lookahead sizes
* must match the settings used when the data was compressed.)
* Returns NULL on error. */
heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size,
uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2);
/* Free a decoder. */
void heatshrink_decoder_free(heatshrink_decoder *hsd);
#endif
/* Reset a decoder. */
void heatshrink_decoder_reset(heatshrink_decoder *hsd);
/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to
* indicate how many bytes were actually sunk (in case a buffer was filled). */
HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd,
uint8_t *in_buf, size_t size, size_t *input_size);
/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into
* OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */
HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size);
/* Notify the dencoder that the input stream is finished.
* If the return value is HSDR_FINISH_MORE, there is still more output, so
* call heatshrink_decoder_poll and repeat. */
HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd);
#endif

641
Modem/compression/heatshrink_encoder.c Normal file
View File

@ -0,0 +1,641 @@
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include "heatshrink_encoder.h"
#include "cfg/debug.h" // Debug configuration from BertOS
typedef enum {
HSES_NOT_FULL, /* input buffer not full enough */
HSES_FILLED, /* buffer is full */
HSES_SEARCH, /* searching for patterns */
HSES_YIELD_TAG_BIT, /* yield tag bit */
HSES_YIELD_LITERAL, /* emit literal byte */
HSES_YIELD_BR_INDEX, /* yielding backref index */
HSES_YIELD_BR_LENGTH, /* yielding backref length */
HSES_SAVE_BACKLOG, /* copying buffer to backlog */
HSES_FLUSH_BITS, /* flush bit buffer */
HSES_DONE, /* done */
} HSE_state;
#if HEATSHRINK_DEBUGGING_LOGS
#include <stdio.h>
#include <ctype.h>
#include <assert.h>
#define LOG(...) fprintf(stderr, __VA_ARGS__)
#define ASSERT(X) assert(X)
static const char *state_names[] = {
"not_full",
"filled",
"search",
"yield_tag_bit",
"yield_literal",
"yield_br_index",
"yield_br_length",
"save_backlog",
"flush_bits",
"done",
};
#else
#define LOG(...) /* no-op */
#define ASSERT(X) /* no-op */
#endif
// Encoder flags
enum {
FLAG_IS_FINISHING = 0x01,
FLAG_HAS_LITERAL = 0x02,
FLAG_ON_FINAL_LITERAL = 0x04,
FLAG_BACKLOG_IS_PARTIAL = 0x08,
FLAG_BACKLOG_IS_FILLED = 0x10,
};
typedef struct {
uint8_t *buf; /* output buffer */
size_t buf_size; /* buffer size */
size_t *output_size; /* bytes pushed to buffer, so far */
} output_info;
#define MATCH_NOT_FOUND ((uint16_t)-1)
static uint16_t get_input_offset(heatshrink_encoder *hse);
static uint16_t get_input_buffer_size(heatshrink_encoder *hse);
static uint16_t get_lookahead_size(heatshrink_encoder *hse);
static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag);
static int can_take_byte(output_info *oi);
static int is_finishing(heatshrink_encoder *hse);
static int backlog_is_partial(heatshrink_encoder *hse);
static int backlog_is_filled(heatshrink_encoder *hse);
static int on_final_literal(heatshrink_encoder *hse);
static void save_backlog(heatshrink_encoder *hse);
static int has_literal(heatshrink_encoder *hse);
/* Push COUNT (max 8) bits to the output buffer, which has room. */
static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits,
output_info *oi);
static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi);
static void push_literal_byte(heatshrink_encoder *hse, output_info *oi);
#if HEATSHRINK_DYNAMIC_ALLOC
heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2,
uint8_t lookahead_sz2) {
if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) ||
(window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) ||
(lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) ||
(lookahead_sz2 > window_sz2)) {
return NULL;
}
/* Note: 2 * the window size is used because the buffer needs to fit
* (1 << window_sz2) bytes for the current input, and an additional
* (1 << window_sz2) bytes for the previous buffer of input, which
* will be scanned for useful backreferences. */
size_t buf_sz = (2 << window_sz2);
kprintf("Trying to allocate: %d\n", buf_sz);
heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz);
if (hse == NULL) { return NULL; }
hse->window_sz2 = window_sz2;
hse->lookahead_sz2 = lookahead_sz2;
heatshrink_encoder_reset(hse);
#if HEATSHRINK_USE_INDEX
size_t index_sz = buf_sz*sizeof(uint16_t);
hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index));
if (hse->search_index == NULL) {
HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz);
return NULL;
}
hse->search_index->size = index_sz;
#endif
LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n",
buf_sz, get_input_buffer_size(hse));
return hse;
}
void heatshrink_encoder_free(heatshrink_encoder *hse) {
size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse));
#if HEATSHRINK_USE_INDEX
size_t index_sz = sizeof(struct hs_index) + hse->search_index->size;
HEATSHRINK_FREE(hse->search_index, index_sz);
(void)index_sz;
#endif
HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz);
(void)buf_sz;
}
#endif
void heatshrink_encoder_reset(heatshrink_encoder *hse) {
size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse));
memset(hse->buffer, 0, buf_sz);
hse->input_size = 0;
hse->state = HSES_NOT_FULL;
hse->match_scan_index = 0;
hse->flags = 0;
hse->bit_index = 0x80;
hse->current_byte = 0x00;
hse->match_length = 0;
hse->outgoing_bits = 0x0000;
hse->outgoing_bits_count = 0;
#ifdef LOOP_DETECT
hse->loop_detect = (uint32_t)-1;
#endif
}
HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse,
uint8_t *in_buf, size_t size, size_t *input_size) {
if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) {
return HSER_SINK_ERROR_NULL;
}
/* Sinking more content after saying the content is done, tsk tsk */
if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; }
/* Sinking more content before processing is done */
if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; }
uint16_t write_offset = get_input_offset(hse) + hse->input_size;
uint16_t ibs = get_input_buffer_size(hse);
uint16_t rem = ibs - hse->input_size;
uint16_t cp_sz = rem < size ? rem : size;
memcpy(&hse->buffer[write_offset], in_buf, cp_sz);
*input_size = cp_sz;
hse->input_size += cp_sz;
LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n",
cp_sz, size, write_offset, hse->input_size);
if (cp_sz == rem) {
LOG("-- internal buffer is now full\n");
hse->state = HSES_FILLED;
}
return HSER_SINK_OK;
}
/***************
* Compression *
***************/
static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start,
uint16_t end, const uint16_t maxlen, uint16_t *match_length);
static void do_indexing(heatshrink_encoder *hse);
static HSE_state st_step_search(heatshrink_encoder *hse);
static HSE_state st_yield_tag_bit(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_yield_literal(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_yield_br_index(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_yield_br_length(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_save_backlog(heatshrink_encoder *hse);
static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse,
output_info *oi);
HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size) {
if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) {
return HSER_POLL_ERROR_NULL;
}
if (out_buf_size == 0) {
LOG("-- MISUSE: output buffer size is 0\n");
return HSER_POLL_ERROR_MISUSE;
}
*output_size = 0;
output_info oi;
oi.buf = out_buf;
oi.buf_size = out_buf_size;
oi.output_size = output_size;
while (1) {
LOG("-- polling, state %u (%s), flags 0x%02x\n",
hse->state, state_names[hse->state], hse->flags);
uint8_t in_state = hse->state;
switch (in_state) {
case HSES_NOT_FULL:
return HSER_POLL_EMPTY;
case HSES_FILLED:
do_indexing(hse);
hse->state = HSES_SEARCH;
break;
case HSES_SEARCH:
hse->state = st_step_search(hse);
break;
case HSES_YIELD_TAG_BIT:
hse->state = st_yield_tag_bit(hse, &oi);
break;
case HSES_YIELD_LITERAL:
hse->state = st_yield_literal(hse, &oi);
break;
case HSES_YIELD_BR_INDEX:
hse->state = st_yield_br_index(hse, &oi);
break;
case HSES_YIELD_BR_LENGTH:
hse->state = st_yield_br_length(hse, &oi);
break;
case HSES_SAVE_BACKLOG:
hse->state = st_save_backlog(hse);
break;
case HSES_FLUSH_BITS:
hse->state = st_flush_bit_buffer(hse, &oi);
case HSES_DONE:
return HSER_POLL_EMPTY;
default:
LOG("-- bad state %s\n", state_names[hse->state]);
return HSER_POLL_ERROR_MISUSE;
}
if (hse->state == in_state) {
/* Check if output buffer is exhausted. */
if (*output_size == out_buf_size) return HSER_POLL_MORE;
}
}
}
HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) {
if (hse == NULL) { return HSER_FINISH_ERROR_NULL; }
LOG("-- setting is_finishing flag\n");
hse->flags |= FLAG_IS_FINISHING;
if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; }
return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE;
}
static HSE_state st_step_search(heatshrink_encoder *hse) {
uint16_t window_length = get_input_buffer_size(hse);
uint16_t lookahead_sz = get_lookahead_size(hse);
uint16_t msi = hse->match_scan_index;
LOG("## step_search, scan @ +%d (%d/%d), input size %d\n",
msi, hse->input_size + msi, 2*window_length, hse->input_size);
bool fin = is_finishing(hse);
if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) {
/* Current search buffer is exhausted, copy it into the
* backlog and await more input. */
LOG("-- end of search @ %d, saving backlog\n", msi);
return HSES_SAVE_BACKLOG;
}
uint16_t input_offset = get_input_offset(hse);
uint16_t end = input_offset + msi;
uint16_t start = 0;
if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */
start = end - window_length + 1;
} else if (backlog_is_partial(hse)) { /* clamp to available data */
start = end - window_length + 1;
if (start < lookahead_sz) { start = lookahead_sz; }
} else { /* only scan available input */
start = input_offset;
}
uint16_t max_possible = lookahead_sz;
if (hse->input_size - msi < lookahead_sz) {
max_possible = hse->input_size - msi;
}
uint16_t match_length = 0;
uint16_t match_pos = find_longest_match(hse,
start, end, max_possible, &match_length);
if (match_pos == MATCH_NOT_FOUND) {
LOG("ss Match not found\n");
hse->match_scan_index++;
hse->flags |= FLAG_HAS_LITERAL;
hse->match_length = 0;
return HSES_YIELD_TAG_BIT;
} else {
LOG("ss Found match of %d bytes at %d\n", match_length, match_pos);
hse->match_pos = match_pos;
hse->match_length = match_length;
ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/);
return HSES_YIELD_TAG_BIT;
}
}
static HSE_state st_yield_tag_bit(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
if (hse->match_length == 0) {
add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER);
return HSES_YIELD_LITERAL;
} else {
add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER);
hse->outgoing_bits = hse->match_pos - 1;
hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse);
return HSES_YIELD_BR_INDEX;
}
} else {
return HSES_YIELD_TAG_BIT; /* output is full, continue */
}
}
static HSE_state st_yield_literal(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
push_literal_byte(hse, oi);
hse->flags &= ~FLAG_HAS_LITERAL;
if (on_final_literal(hse)) { return HSES_FLUSH_BITS; }
return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH;
} else {
return HSES_YIELD_LITERAL;
}
}
static HSE_state st_yield_br_index(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
LOG("-- yielding backref index %u\n", hse->match_pos);
if (push_outgoing_bits(hse, oi) > 0) {
return HSES_YIELD_BR_INDEX; /* continue */
} else {
hse->outgoing_bits = hse->match_length - 1;
hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse);
return HSES_YIELD_BR_LENGTH; /* done */
}
} else {
return HSES_YIELD_BR_INDEX; /* continue */
}
}
static HSE_state st_yield_br_length(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
LOG("-- yielding backref length %u\n", hse->match_length);
if (push_outgoing_bits(hse, oi) > 0) {
return HSES_YIELD_BR_LENGTH;
} else {
hse->match_scan_index += hse->match_length;
hse->match_length = 0;
return HSES_SEARCH;
}
} else {
return HSES_YIELD_BR_LENGTH;
}
}
static HSE_state st_save_backlog(heatshrink_encoder *hse) {
if (is_finishing(hse)) {
/* copy remaining literal (if necessary) */
if (has_literal(hse)) {
hse->flags |= FLAG_ON_FINAL_LITERAL;
return HSES_YIELD_TAG_BIT;
} else {
return HSES_FLUSH_BITS;
}
} else {
LOG("-- saving backlog\n");
save_backlog(hse);
return HSES_NOT_FULL;
}
}
static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse,
output_info *oi) {
if (hse->bit_index == 0x80) {
LOG("-- done!\n");
return HSES_DONE;
} else if (can_take_byte(oi)) {
LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index);
oi->buf[(*oi->output_size)++] = hse->current_byte;
LOG("-- done!\n");
return HSES_DONE;
} else {
return HSES_FLUSH_BITS;
}
}
static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) {
LOG("-- adding tag bit: %d\n", tag);
push_bits(hse, 1, tag, oi);
}
static uint16_t get_input_offset(heatshrink_encoder *hse) {
return get_input_buffer_size(hse);
}
static uint16_t get_input_buffer_size(heatshrink_encoder *hse) {
return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse));
(void)hse;
}
static uint16_t get_lookahead_size(heatshrink_encoder *hse) {
return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse));
(void)hse;
}
static void do_indexing(heatshrink_encoder *hse) {
#if HEATSHRINK_USE_INDEX
/* Build an index array I that contains flattened linked lists
* for the previous instances of every byte in the buffer.
*
* For example, if buf[200] == 'x', then index[200] will either
* be an offset i such that buf[i] == 'x', or a negative offset
* to indicate end-of-list. This significantly speeds up matching,
* while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM.
*
* Future optimization options:
* 1. Since any negative value represents end-of-list, the other
* 15 bits could be used to improve the index dynamically.
*
* 2. Likewise, the last lookahead_sz bytes of the index will
* not be usable, so temporary data could be stored there to
* dynamically improve the index.
* */
struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse);
uint16_t last[256];
memset(last, 0xFF, sizeof(last));
uint8_t * const data = hse->buffer;
int16_t * const index = hsi->index;
const uint16_t input_offset = get_input_offset(hse);
const uint16_t end = input_offset + hse->input_size;
for (uint16_t i=0; i<end; i++) {
uint8_t v = data[i];
uint16_t lv = last[v];
index[i] = lv;
last[v] = i;
}
#else
(void)hse;
#endif
}
static int is_finishing(heatshrink_encoder *hse) {
return hse->flags & FLAG_IS_FINISHING;
}
static int backlog_is_partial(heatshrink_encoder *hse) {
return hse->flags & FLAG_BACKLOG_IS_PARTIAL;
}
static int backlog_is_filled(heatshrink_encoder *hse) {
return hse->flags & FLAG_BACKLOG_IS_FILLED;
}
static int on_final_literal(heatshrink_encoder *hse) {
return hse->flags & FLAG_ON_FINAL_LITERAL;
}
static int has_literal(heatshrink_encoder *hse) {
return (hse->flags & FLAG_HAS_LITERAL);
}
static int can_take_byte(output_info *oi) {
return *oi->output_size < oi->buf_size;
}
/* Return the longest match for the bytes at buf[end:end+maxlen] between
* buf[start] and buf[end-1]. If no match is found, return -1. */
static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start,
uint16_t end, const uint16_t maxlen, uint16_t *match_length) {
LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n",
end, end + maxlen, start, end + maxlen - 1, maxlen);
uint8_t *buf = hse->buffer;
uint16_t match_maxlen = 0;
uint16_t match_index = MATCH_NOT_FOUND;
const uint16_t break_even_point = 3;
uint16_t len = 0;
uint8_t * const needlepoint = &buf[end];
#if HEATSHRINK_USE_INDEX
struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse);
int16_t pos = hsi->index[end];
while (pos >= start) {
uint8_t * const pospoint = &buf[pos];
len = 0;
for (len = 1; len < maxlen; len++) {
if (pospoint[len] != needlepoint[len]) break;
}
if (len > match_maxlen) {
match_maxlen = len;
match_index = pos;
if (len == maxlen) { break; } /* won't find better */
}
pos = hsi->index[pos];
}
#else
for (int16_t pos=end - 1; pos >= start; pos--) {
for (len=0; len<maxlen; len++) {
if (0) {
LOG(" --> cmp buf[%d] == 0x%02x against %02x (start %u)\n",
pos + len, buf[pos + len], needlepoint[len], start);
}
if (buf[pos + len] != needlepoint[len]) { break; }
}
if (len > match_maxlen) {
match_maxlen = len;
match_index = pos;
if (len == maxlen) { break; } /* don't keep searching */
}
}
#endif
if (match_maxlen >= break_even_point) {
LOG("-- best match: %u bytes at -%u\n",
match_maxlen, end - match_index);
*match_length = match_maxlen;
return end - match_index;
}
LOG("-- none found\n");
return MATCH_NOT_FOUND;
}
static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) {
uint8_t count = 0;
uint8_t bits = 0;
if (hse->outgoing_bits_count > 8) {
count = 8;
bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8);
} else {
count = hse->outgoing_bits_count;
bits = hse->outgoing_bits;
}
if (count > 0) {
LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits);
push_bits(hse, count, bits, oi);
hse->outgoing_bits_count -= count;
}
return count;
}
/* Push COUNT (max 8) bits to the output buffer, which has room.
* Bytes are set from the lowest bits, up. */
static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits,
output_info *oi) {
ASSERT(count <= 8);
LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits);
/* If adding a whole byte and at the start of a new output byte,
* just push it through whole and skip the bit IO loop. */
if (count == 8 && hse->bit_index == 0x80) {
oi->buf[(*oi->output_size)++] = bits;
} else {
for (int i=count - 1; i>=0; i--) {
bool bit = bits & (1 << i);
if (bit) { hse->current_byte |= hse->bit_index; }
if (0) {
LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n",
bit ? 1 : 0, hse->bit_index, hse->current_byte);
}
hse->bit_index >>= 1;
if (hse->bit_index == 0x00) {
hse->bit_index = 0x80;
LOG(" > pushing byte 0x%02x\n", hse->current_byte);
oi->buf[(*oi->output_size)++] = hse->current_byte;
hse->current_byte = 0x00;
}
}
}
}
static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) {
uint16_t processed_offset = hse->match_scan_index - 1;
uint16_t input_offset = get_input_offset(hse) + processed_offset;
uint8_t c = hse->buffer[input_offset];
LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n",
c, isprint(c) ? c : '.', input_offset);
push_bits(hse, 8, c, oi);
}
static void save_backlog(heatshrink_encoder *hse) {
size_t input_buf_sz = get_input_buffer_size(hse);
uint16_t msi = hse->match_scan_index;
/* Copy processed data to beginning of buffer, so it can be
* used for future matches. Don't bother checking whether the
* input is less than the maximum size, because if it isn't,
* we're done anyway. */
uint16_t rem = input_buf_sz - msi; // unprocessed bytes
uint16_t shift_sz = input_buf_sz + rem;
memmove(&hse->buffer[0],
&hse->buffer[input_buf_sz - rem],
shift_sz);
if (backlog_is_partial(hse)) {
/* The whole backlog is filled in now, so include it in scans. */
hse->flags |= FLAG_BACKLOG_IS_FILLED;
} else {
/* Include backlog, except for the first lookahead_sz bytes, which
* are still undefined. */
hse->flags |= FLAG_BACKLOG_IS_PARTIAL;
}
hse->match_scan_index = 0;
hse->input_size -= input_buf_sz - rem;
}

109
Modem/compression/heatshrink_encoder.h Normal file
View File

@ -0,0 +1,109 @@
#ifndef HEATSHRINK_ENCODER_H
#define HEATSHRINK_ENCODER_H
#include <stdint.h>
#include <stddef.h>
#include "heatshrink_common.h"
#include "heatshrink_config.h"
typedef enum {
HSER_SINK_OK, /* data sunk into input buffer */
HSER_SINK_ERROR_NULL=-1, /* NULL argument */
HSER_SINK_ERROR_MISUSE=-2, /* API misuse */
} HSE_sink_res;
typedef enum {
HSER_POLL_EMPTY, /* input exhausted */
HSER_POLL_MORE, /* poll again for more output */
HSER_POLL_ERROR_NULL=-1, /* NULL argument */
HSER_POLL_ERROR_MISUSE=-2, /* API misuse */
} HSE_poll_res;
typedef enum {
HSER_FINISH_DONE, /* encoding is complete */
HSER_FINISH_MORE, /* more output remaining; use poll */
HSER_FINISH_ERROR_NULL=-1, /* NULL argument */
} HSE_finish_res;
#if HEATSHRINK_DYNAMIC_ALLOC
#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \
((HSE)->window_sz2)
#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \
((HSE)->lookahead_sz2)
#define HEATSHRINK_ENCODER_INDEX(HSE) \
((HSE)->search_index)
struct hs_index {
uint16_t size;
int16_t index[];
};
#else
#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \
(HEATSHRINK_STATIC_WINDOW_BITS)
#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \
(HEATSHRINK_STATIC_LOOKAHEAD_BITS)
#define HEATSHRINK_ENCODER_INDEX(HSE) \
(&(HSE)->search_index)
struct hs_index {
uint16_t size;
int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS];
};
#endif
typedef struct {
uint16_t input_size; /* bytes in input buffer */
uint16_t match_scan_index;
uint16_t match_length;
uint16_t match_pos;
uint16_t outgoing_bits; /* enqueued outgoing bits */
uint8_t outgoing_bits_count;
uint8_t flags;
uint8_t state; /* current state machine node */
uint8_t current_byte; /* current byte of output */
uint8_t bit_index; /* current bit index */
#if HEATSHRINK_DYNAMIC_ALLOC
uint8_t window_sz2; /* 2^n size of window */
uint8_t lookahead_sz2; /* 2^n size of lookahead */
#if HEATSHRINK_USE_INDEX
struct hs_index *search_index;
#endif
/* input buffer and / sliding window for expansion */
uint8_t buffer[];
#else
#if HEATSHRINK_USE_INDEX
struct hs_index search_index;
#endif
/* input buffer and / sliding window for expansion */
uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)];
#endif
} heatshrink_encoder;
#if HEATSHRINK_DYNAMIC_ALLOC
/* Allocate a new encoder struct and its buffers.
* Returns NULL on error. */
heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2,
uint8_t lookahead_sz2);
/* Free an encoder. */
void heatshrink_encoder_free(heatshrink_encoder *hse);
#endif
/* Reset an encoder. */
void heatshrink_encoder_reset(heatshrink_encoder *hse);
/* Sink up to SIZE bytes from IN_BUF into the encoder.
* INPUT_SIZE is set to the number of bytes actually sunk (in case a
* buffer was filled.). */
HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse,
uint8_t *in_buf, size_t size, size_t *input_size);
/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into
* OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */
HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size);
/* Notify the encoder that the input stream is finished.
* If the return value is HSER_FINISH_MORE, there is still more output, so
* call heatshrink_encoder_poll and repeat. */
HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse);
#endif

362
Modem/compression/lzfx.c
View File

@ -1,362 +0,0 @@
/*
* Copyright (c) 2009 Andrew Collette <andrew.collette at gmail.com>
* http://lzfx.googlecode.com
*
* Implements an LZF-compatible compressor/decompressor based on the liblzf
* codebase written by Marc Lehmann. This code is released under the BSD
* license. License and original copyright statement follow.
*
*
* Copyright (c) 2000-2008 Marc Alexander Lehmann <schmorp@schmorp.de>
*
* Redistribution and use in source and binary forms, with or without modifica-
* tion, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
* CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
* CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
* ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "lzfx.h"
#define LZFX_HSIZE (1 << (LZFX_HLOG))
/* We need this for memset */
# include <string.h>
// #if __GNUC__ >= 3 && !DISABLE_EXPECT
// # define fx_expect_false(expr) __builtin_expect((expr) != 0, 0)
// # define fx_expect_true(expr) __builtin_expect((expr) != 0, 1)
// #else
# define fx_expect_false(expr) (expr)
# define fx_expect_true(expr) (expr)
//#endif
typedef unsigned char u8;
typedef const u8 *LZSTATE[LZFX_HSIZE];
/* Define the hash function */
#define LZFX_FRST(p) (((p[0]) << 8) | p[1])
#define LZFX_NEXT(v,p) (((v) << 8) | p[2])
#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1))
/* These cannot be changed, as they are related to the compressed format. */
#define LZFX_MAX_LIT (1 << 5)
#define LZFX_MAX_OFF (1 << 13)
#define LZFX_MAX_REF ((1 << 8) + (1 << 3))
static
int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen);
/* Compressed format
There are two kinds of structures in LZF/LZFX: literal runs and back
references. The length of a literal run is encoded as L - 1, as it must
contain at least one byte. Literals are encoded as follows:
000LLLLL <L+1 bytes>
Back references are encoded as follows. The smallest possible encoded
length value is 1, as otherwise the control byte would be recognized as
a literal run. Since at least three bytes must match for a back reference
to be inserted, the length is encoded as L - 2 instead of L - 1. The
offset (distance to the desired data in the output buffer) is encoded as
o - 1, as all offsets are at least 1. The binary format is:
LLLooooo oooooooo for backrefs of real length < 9 (1 <= L < 7)
111ooooo LLLLLLLL oooooooo for backrefs of real length >= 9 (L > 7)
*/
#include <stdio.h>
int lzfx_compress(const void *const ibuf, const unsigned int ilen,
void *obuf, unsigned int *const olen){
/* Hash table; an array of u8*'s which point
to various locations in the input buffer */
const u8 *htab[LZFX_HSIZE];
const u8 **hslot; /* Pointer to entry in hash table */
unsigned int hval; /* Hash value generated by macros above */
const u8 *ref; /* Pointer to candidate match location in input */
const u8 *ip = (const u8 *)ibuf;
const u8 *const in_end = ip + ilen;
u8 *op = (u8 *)obuf;
const u8 *const out_end = (olen == NULL ? NULL : op + *olen);
int lit; /* # of bytes in current literal run */
#if defined (WIN32) && defined (_M_X64)
unsigned _int64 off; /* workaround for missing POSIX compliance */
#else
unsigned long off;
#endif
if(olen == NULL) return LZFX_EARGS;
if(ibuf == NULL){
if(ilen != 0) return LZFX_EARGS;
*olen = 0;
return 0;
}
if(obuf == NULL){
if(olen != 0) return LZFX_EARGS;
return lzfx_getsize(ibuf, ilen, olen);
}
memset(htab, 0, sizeof(htab));
/* Start a literal run. Whenever we do this the output pointer is
advanced because the current byte will hold the encoded length. */
lit = 0; op++;
hval = LZFX_FRST(ip);
while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */
hval = LZFX_NEXT(hval, ip);
hslot = htab + LZFX_IDX(hval);
ref = *hslot; *hslot = ip;
if( ref < ip
&& (off = ip - ref - 1) < LZFX_MAX_OFF
&& ip + 4 < in_end /* Backref takes up to 3 bytes, so don't bother */
&& ref > (u8 *)ibuf
&& ref[0] == ip[0]
&& ref[1] == ip[1]
&& ref[2] == ip[2] ) {
unsigned int len = 3; /* We already know 3 bytes match */
const unsigned int maxlen = in_end - ip - 2 > LZFX_MAX_REF ?
LZFX_MAX_REF : in_end - ip - 2;
/* lit == 0: op + 3 must be < out_end (because we undo the run)
lit != 0: op + 3 + 1 must be < out_end */
if(fx_expect_false(op - !lit + 3 + 1 >= out_end))
return LZFX_ESIZE;
op [- lit - 1] = lit - 1; /* Terminate literal run */
op -= !lit; /* Undo run if length is zero */
/* Start checking at the fourth byte */
while (len < maxlen && ref[len] == ip[len])
len++;
len -= 2; /* We encode the length as #octets - 2 */
/* Format 1: [LLLooooo oooooooo] */
if (len < 7) {
*op++ = (off >> 8) + (len << 5);
*op++ = off;
/* Format 2: [111ooooo LLLLLLLL oooooooo] */
} else {
*op++ = (off >> 8) + (7 << 5);
*op++ = len - 7;
*op++ = off;
}
lit = 0; op++;
ip += len + 1; /* ip = initial ip + #octets -1 */
if (fx_expect_false (ip + 3 >= in_end)){
ip++; /* Code following expects exit at bottom of loop */
break;
}
hval = LZFX_FRST (ip);
hval = LZFX_NEXT (hval, ip);
htab[LZFX_IDX (hval)] = ip;
ip++; /* ip = initial ip + #octets */
} else {
/* Keep copying literal bytes */
if (fx_expect_false (op >= out_end)) return LZFX_ESIZE;
lit++; *op++ = *ip++;
if (fx_expect_false (lit == LZFX_MAX_LIT)) {
op [- lit - 1] = lit - 1; /* stop run */
lit = 0; op++; /* start run */
}
} /* if() found match in htab */
} /* while(ip < ilen -2) */
/* At most 3 bytes remain in input. We therefore need 4 bytes available
in the output buffer to store them (3 data + ctrl byte).*/
if (op + 3 > out_end) return LZFX_ESIZE;
while (ip < in_end) {
lit++; *op++ = *ip++;
if (fx_expect_false (lit == LZFX_MAX_LIT)){
op [- lit - 1] = lit - 1;
lit = 0; op++;
}
}
op [- lit - 1] = lit - 1;
op -= !lit;
*olen = op - (u8 *)obuf;
return 0;
}
/* Decompressor */
int lzfx_decompress(const void* ibuf, unsigned int ilen,
void* obuf, unsigned int *olen){
u8 const *ip = (const u8 *)ibuf;
u8 const *const in_end = ip + ilen;
u8 *op = (u8 *)obuf;
u8 const *const out_end = (olen == NULL ? NULL : op + *olen);
unsigned int remain_len = 0;
int rc;
if(olen == NULL) return LZFX_EARGS;
if(ibuf == NULL){
if(ilen != 0) return LZFX_EARGS;
*olen = 0;
return 0;
}
if(obuf == NULL){
if(olen != 0) return LZFX_EARGS;
return lzfx_getsize(ibuf, ilen, olen);
}
do {
unsigned int ctrl = *ip++;
/* Format 000LLLLL: a literal byte string follows, of length L+1 */
if(ctrl < (1 << 5)) {
ctrl++;
if(fx_expect_false(op + ctrl > out_end)){
--ip; /* Rewind to control byte */
goto guess;
}
if(fx_expect_false(ip + ctrl > in_end)) return LZFX_ECORRUPT;
do
*op++ = *ip++;
while(--ctrl);
/* Format #1 [LLLooooo oooooooo]: backref of length L+1+2
^^^^^ ^^^^^^^^
A B
#2 [111ooooo LLLLLLLL oooooooo] backref of length L+7+2
^^^^^ ^^^^^^^^
A B
In both cases the location of the backref is computed from the
remaining part of the data as follows:
location = op - A*256 - B - 1
*/
} else {
unsigned int len = (ctrl >> 5);
u8 *ref = op - ((ctrl & 0x1f) << 8) -1;
if(len==7) len += *ip++; /* i.e. format #2 */
len += 2; /* len is now #octets */
if(fx_expect_false(op + len > out_end)){
ip -= (len >= 9) ? 2 : 1; /* Rewind to control byte */
goto guess;
}
if(fx_expect_false(ip >= in_end)) return LZFX_ECORRUPT;
ref -= *ip++;
if(fx_expect_false(ref < (u8*)obuf)) return LZFX_ECORRUPT;
do
*op++ = *ref++;
while (--len);
}
} while (ip < in_end);
*olen = op - (u8 *)obuf;
return 0;
guess:
rc = lzfx_getsize(ip, ilen - (ip-(u8*)ibuf), &remain_len);
if(rc>=0) *olen = remain_len + (op - (u8*)obuf);
return rc;
}
/* Guess len. No parameters may be NULL; this is not checked. */
static
int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen){
u8 const *ip = (const u8 *)ibuf;
u8 const *const in_end = ip + ilen;
int tot_len = 0;
while (ip < in_end) {
unsigned int ctrl = *ip++;
if(ctrl < (1 << 5)) {
ctrl++;
if(ip + ctrl > in_end)
return LZFX_ECORRUPT;
tot_len += ctrl;
ip += ctrl;
} else {
unsigned int len = (ctrl >> 5);
if(len==7){ /* i.e. format #2 */
len += *ip++;
}
len += 2; /* len is now #octets */
if(ip >= in_end) return LZFX_ECORRUPT;
ip++; /* skip the ref byte */
tot_len += len;
}
}
*olen = tot_len;
return 0;
}

90
Modem/compression/lzfx.h
View File

@ -1,90 +0,0 @@
/*
* Copyright (c) 2009 Andrew Collette <andrew.collette at gmail.com>
* http://lzfx.googlecode.com
*
* Implements an LZF-compatible compressor/decompressor based on the liblzf
* codebase written by Marc Lehmann. This code is released under the BSD
* license. License and original copyright statement follow.
*
*
* Copyright (c) 2000-2008 Marc Alexander Lehmann <schmorp@schmorp.de>
*
* Redistribution and use in source and binary forms, with or without modifica-
* tion, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER-
* CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE-
* CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH-
* ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LZFX_H
#define LZFX_H
/* Documented behavior, including function signatures and error codes,
is guaranteed to remain unchanged for releases with the same major
version number. Releases of the same major version are also able
to read each other's output, although the output itself is not
guaranteed to be byte-for-byte identical.
*/
#define LZFX_VERSION_MAJOR 0
#define LZFX_VERSION_MINOR 1
#define LZFX_VERSION_STRING "0.1"
/* Hashtable size (2**LZFX_HLOG entries) */
#ifndef LZFX_HLOG
# define LZFX_HLOG 8
#endif
/* Predefined errors. */
#define LZFX_ESIZE -1 /* Output buffer too small */
#define LZFX_ECORRUPT -2 /* Invalid data for decompression */
#define LZFX_EARGS -3 /* Arguments invalid (NULL) */
/* Buffer-to buffer compression.
Supply pre-allocated input and output buffers via ibuf and obuf, and
their size in bytes via ilen and olen. Buffers may not overlap.
On success, the function returns a non-negative value and the argument
olen contains the compressed size in bytes. On failure, a negative
value is returned and olen is not modified.
*/
int lzfx_compress(const void* ibuf, unsigned int ilen,
void* obuf, unsigned int *olen);
/* Buffer-to-buffer decompression.
Supply pre-allocated input and output buffers via ibuf and obuf, and
their size in bytes via ilen and olen. Buffers may not overlap.
On success, the function returns a non-negative value and the argument
olen contains the uncompressed size in bytes. On failure, a negative
value is returned.
If the failure code is LZFX_ESIZE, olen contains the minimum buffer size
required to hold the decompressed data. Otherwise, olen is not modified.
Supplying a zero *olen is a valid and supported strategy to determine the
required buffer size. This does not require decompression of the entire
stream and is consequently very fast. Argument obuf may be NULL in
this case only.
*/
int lzfx_decompress(const void* ibuf, unsigned int ilen,
void* obuf, unsigned int *olen);
#endif

120
Modem/main.c
View File

@ -15,6 +15,8 @@
#include "afsk.h" // Header for AFSK modem
#include "protocol/mp1.h" // Header for MP.1 protocol
#include "compression/heatshrink_encoder.h"
#include "compression/heatshrink_decoder.h"
//////////////////////////////////////////////////////
// A few definitions //
@ -39,6 +41,114 @@ static int serialLen = 0; // Counter for counting length of data from seri
static bool sertx = false; // Flag signifying whether it's time to send data
// Received on the serial port.
static uint8_t compressedData[MP1_MAX_FRAME_LENGTH];
static uint8_t decompressedData[MP1_MAX_FRAME_LENGTH];
static int freeRam () {
extern int __heap_start, *__brkval;
int v;
return (int) &v - (__brkval == 0 ? (int) &__heap_start : (int) __brkval);
}
static size_t compress(uint8_t *input, size_t length) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4);
if (hse == NULL) {
kprintf("Could not allocate encoder");
return 0;
}
size_t written = 0;
size_t sunk = 0;
heatshrink_encoder_sink(hse, input, length, &sunk);
int status = heatshrink_encoder_finish(hse);
if (sunk < length) {
kprintf("Not all data was sunk into encoder\n");
heatshrink_encoder_free(hse);
return 0;
} else {
kprintf("Bytes sunk into HSE: %d\n", length);
if (status == HSER_FINISH_MORE) {
heatshrink_encoder_poll(hse, compressedData, MP1_MAX_FRAME_LENGTH, &written);
kprintf("Bytes written into buffer: %d\n", written);
} else {
kprintf("All input data was sunk, but encoder doesn't have any data for us.");
}
}
heatshrink_encoder_free(hse);
return written;
}
static size_t decompress(uint8_t *input, size_t length) {
heatshrink_decoder *hsd = heatshrink_decoder_alloc(MP1_MAX_FRAME_LENGTH, 8, 4);
if (hsd == NULL) {
kprintf("Could not allocate decoder");
return 0;
}
kprintf("\nDecoder allocated. Free RAM: %d bytes\n", freeRam());
size_t written = 0;
size_t sunk = 0;
heatshrink_decoder_sink(hsd, input, length, &sunk);
int status = heatshrink_decoder_finish(hsd);
if (sunk < length) {
kprintf("Not all data was sunk into decoder\n");
heatshrink_decoder_free(hsd);
return 0;
} else {
kprintf("Bytes sunk into HSD: %d\n", length);
if (status == HSER_FINISH_MORE) {
heatshrink_decoder_poll(hsd, decompressedData, MP1_MAX_FRAME_LENGTH, &written);
kprintf("Bytes written into decompression buffer: %d\n", written);
} else {
kprintf("All input data was sunk, but the decoder doesn't have any data for us.");
}
}
heatshrink_decoder_free(hsd);
return written;
}
static void hseTest() {
kprintf("\nFree RAM: %d bytes\n", freeRam());
size_t compressed_size = compress(serialBuffer, serialLen);
size_t decompressed_size = decompress(compressedData, compressed_size);
kprintf("\n-------------------\nInput size: %d\nCompressed size: %d\nDecompressed size: %d\n", serialLen, compressed_size, decompressed_size);
// heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4);
// kprintf("\nFree RAM: %d bytes\n", freeRam());
// size_t out_sz = 50;
// uint8_t out_buf[out_sz];
// size_t written = 0;
// kprintf("\nFree RAM: %d bytes\n", freeRam());
// size_t length = serialLen;
// heatshrink_encoder_sink(hse, serialBuffer, serialLen, &length);
// int returnv = heatshrink_encoder_finish(hse);
// kprintf("Encoder finish returned: %d\n", returnv);
// if (length < serialLen) {
// kprintf("Not all data was sunk into encoder\n");
// } else {
// // All data delivered
// kprintf("Bytes sunk into HSE: %d\n", length);
// heatshrink_encoder_poll(hse, out_buf, out_sz, &written);
// kprintf("2: Bytes written into buffer: %d\n", written);
// }
// heatshrink_encoder_free(hse);
}
//////////////////////////////////////////////////////
// And here comes the actual program :) //
@ -52,12 +162,6 @@ static void mp1Callback(struct MP1Packet *packet) {
//kprintf("%.*s\n", packet->dataLength, packet->data);
}
// static int freeRam () {
// extern int __heap_start, *__brkval;
// int v;
// return (int) &v - (__brkval == 0 ? (int) &__heap_start : (int) __brkval);
// }
// Simple initialization function.
static void init(void)
{
@ -125,7 +229,9 @@ int main(void)
if (sertx) {
// If we should, pass the buffer to the protocol's
// send function.
mp1Send(&mp1, serialBuffer, serialLen);
hseTest();
// mp1Send(&mp1, serialBuffer, serialLen);
// Reset the transmission flag and length counter
sertx = false;

2
Modem/protocol/mp1.h
View File

@ -4,8 +4,6 @@
#include <cfg/compiler.h>
#include <io/kfile.h>
#include "compression/lzfx.h"
// Frame sizing & checksum
#define MP1_MIN_FRAME_LENGTH 3
#define MP1_MAX_FRAME_LENGTH 200

2
buildrev.h
View File

@ -1,2 +1,2 @@
#define VERS_BUILD 930
#define VERS_BUILD 994
#define VERS_HOST "vixen"