From 434cf6f2e6b42d00a50b146d8291a487671f419c Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 14 Apr 2014 11:44:39 +0200 Subject: [PATCH] After compression lib experiment --- Modem/compression/fastlz.c | 543 +++++++++++++++++++++++++++++++++++++ Modem/compression/fastlz.h | 100 +++++++ Modem/compression/lzfx.c | 362 +++++++++++++++++++++++++ Modem/compression/lzfx.h | 90 ++++++ Modem/config.h | 2 +- Modem/main.c | 8 + Modem/protocol/mp1.h | 4 +- buildrev.h | 2 +- 8 files changed, 1108 insertions(+), 3 deletions(-) create mode 100644 Modem/compression/fastlz.c create mode 100644 Modem/compression/fastlz.h create mode 100644 Modem/compression/lzfx.c create mode 100644 Modem/compression/lzfx.h diff --git a/Modem/compression/fastlz.c b/Modem/compression/fastlz.c new file mode 100644 index 0000000..4273d51 --- /dev/null +++ b/Modem/compression/fastlz.c @@ -0,0 +1,543 @@ +/* + FastLZ - lightning-fast lossless compression library + + Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) + Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) + Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#if !defined(FASTLZ__COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) + +/* + * Always check for bound when decompressing. + * Generally it is best to leave it defined. + */ +#define FASTLZ_SAFE + +/* + * Give hints to the compiler for branch prediction optimization. + */ +#if defined(__GNUC__) && (__GNUC__ > 2) +#define FASTLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) +#define FASTLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) +#else +#define FASTLZ_EXPECT_CONDITIONAL(c) (c) +#define FASTLZ_UNEXPECT_CONDITIONAL(c) (c) +#endif + +/* + * Use inlined functions for supported systems. + */ +#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C) +#define FASTLZ_INLINE inline +#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__) +#define FASTLZ_INLINE __inline +#else +#define FASTLZ_INLINE +#endif + +/* + * Prevent accessing more than 8-bit at once, except on x86 architectures. + */ +#if !defined(FASTLZ_STRICT_ALIGN) +#define FASTLZ_STRICT_ALIGN +#if defined(__i386__) || defined(__386) /* GNU C, Sun Studio */ +#undef FASTLZ_STRICT_ALIGN +#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */ +#undef FASTLZ_STRICT_ALIGN +#elif defined(_M_IX86) /* Intel, MSVC */ +#undef FASTLZ_STRICT_ALIGN +#elif defined(__386) +#undef FASTLZ_STRICT_ALIGN +#elif defined(_X86_) /* MinGW */ +#undef FASTLZ_STRICT_ALIGN +#elif defined(__I86__) /* Digital Mars */ +#undef FASTLZ_STRICT_ALIGN +#endif +#endif + +/* + * FIXME: use preprocessor magic to set this on different platforms! + */ +typedef unsigned char flzuint8; +typedef unsigned short flzuint16; +typedef unsigned int flzuint32; + +/* prototypes */ +int fastlz_compress(const void* input, int length, void* output); +int fastlz_compress_level(int level, const void* input, int length, void* output); +int fastlz_decompress(const void* input, int length, void* output, int maxout); + +#define MAX_COPY 32 +#define MAX_LEN 264 /* 256 + 8 */ +#define MAX_DISTANCE 8192 + +#if !defined(FASTLZ_STRICT_ALIGN) +#define FASTLZ_READU16(p) *((const flzuint16*)(p)) +#else +#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8) +#endif + +#define HASH_LOG 13 +#define HASH_SIZE (1<< HASH_LOG) +#define HASH_MASK (HASH_SIZE-1) +#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; } + +#undef FASTLZ_LEVEL +#define FASTLZ_LEVEL 1 + +#undef FASTLZ_COMPRESSOR +#undef FASTLZ_DECOMPRESSOR +#define FASTLZ_COMPRESSOR fastlz1_compress +#define FASTLZ_DECOMPRESSOR fastlz1_decompress +static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); +static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); +#include "fastlz.c" + +#undef FASTLZ_LEVEL +#define FASTLZ_LEVEL 2 + +#undef MAX_DISTANCE +#define MAX_DISTANCE 8191 +#define MAX_FARDISTANCE (65535+MAX_DISTANCE-1) + +#undef FASTLZ_COMPRESSOR +#undef FASTLZ_DECOMPRESSOR +#define FASTLZ_COMPRESSOR fastlz2_compress +#define FASTLZ_DECOMPRESSOR fastlz2_decompress +static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output); +static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout); +#include "fastlz.c" + +int fastlz_compress(const void* input, int length, void* output) +{ + return fastlz1_compress(input, length, output); + + // Fastlz2 doesn't really matter for us + // because of small block sizes + // return fastlz2_compress(input, length, output); +} + +int fastlz_decompress(const void* input, int length, void* output, int maxout) +{ + /* magic identifier for compression level */ + int level = ((*(const flzuint8*)input) >> 5) + 1; + + if(level == 1) + return fastlz1_decompress(input, length, output, maxout); + if(level == 2) + return fastlz2_decompress(input, length, output, maxout); + + /* unknown level, trigger error */ + return 0; +} + +int fastlz_compress_level(int level, const void* input, int length, void* output) +{ + if(level == 1) + return fastlz1_compress(input, length, output); + if(level == 2) + return fastlz2_compress(input, length, output); + + return 0; +} + +#else /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ + +static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output) +{ + const flzuint8* ip = (const flzuint8*) input; + const flzuint8* ip_bound = ip + length - 2; + const flzuint8* ip_limit = ip + length - 12; + flzuint8* op = (flzuint8*) output; + + const flzuint8* htab[HASH_SIZE]; + const flzuint8** hslot; + flzuint32 hval; + + flzuint32 copy; + + /* sanity check */ + if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4)) + { + if(length) + { + /* create literal copy only */ + *op++ = length-1; + ip_bound++; + while(ip <= ip_bound) + *op++ = *ip++; + return length+1; + } + else + return 0; + } + + /* initializes hash table */ + for (hslot = htab; hslot < htab + HASH_SIZE; hslot++) + *hslot = ip; + + /* we start with literal copy */ + copy = 2; + *op++ = MAX_COPY-1; + *op++ = *ip++; + *op++ = *ip++; + + /* main loop */ + while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) + { + const flzuint8* ref; + flzuint32 distance; + + /* minimum match length */ + flzuint32 len = 3; + + /* comparison starting-point */ + const flzuint8* anchor = ip; + + /* check for a run */ +#if FASTLZ_LEVEL==2 + if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1)) + { + distance = 1; + ip += 3; + ref = anchor - 1 + 3; + goto match; + } +#endif + + /* find potential match */ + HASH_FUNCTION(hval,ip); + hslot = htab + hval; + ref = htab[hval]; + + /* calculate distance to the match */ + distance = anchor - ref; + + /* update hash table */ + *hslot = anchor; + + /* is this a match? check the first 3 bytes */ + if(distance==0 || (distance >= MAX_DISTANCE) || *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++) goto literal; + +#if FASTLZ_LEVEL==2 + /* far, needs at least 5-byte match */ + if(distance >= MAX_DISTANCE) + { + if(*ip++ != *ref++ || *ip++!= *ref++) + goto literal; + len += 2; + } + + match: +#endif + + /* last matched byte */ + ip = anchor + len; + + /* distance is biased */ + distance--; + + if(!distance) + { + /* zero distance means a run */ + flzuint8 x = ip[-1]; + while(ip < ip_bound) + if(*ref++ != x) break; else ip++; + } + else + for(;;) + { + /* safe because the outer check against ip limit */ + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + if(*ref++ != *ip++) break; + while(ip < ip_bound) + if(*ref++ != *ip++) break; + break; + } + + /* if we have copied something, adjust the copy count */ + if(copy) + /* copy is biased, '0' means 1 byte copy */ + *(op-copy-1) = copy-1; + else + /* back, to overwrite the copy count */ + op--; + + /* reset literal counter */ + copy = 0; + + /* length is biased, '1' means a match of 3 bytes */ + ip -= 3; + len = ip - anchor; + + /* encode the match */ +#if FASTLZ_LEVEL==2 + if(distance < MAX_DISTANCE) + { + if(len < 7) + { + *op++ = (len << 5) + (distance >> 8); + *op++ = (distance & 255); + } + else + { + *op++ = (7 << 5) + (distance >> 8); + for(len-=7; len >= 255; len-= 255) + *op++ = 255; + *op++ = len; + *op++ = (distance & 255); + } + } + else + { + /* far away, but not yet in the another galaxy... */ + if(len < 7) + { + distance -= MAX_DISTANCE; + *op++ = (len << 5) + 31; + *op++ = 255; + *op++ = distance >> 8; + *op++ = distance & 255; + } + else + { + distance -= MAX_DISTANCE; + *op++ = (7 << 5) + 31; + for(len-=7; len >= 255; len-= 255) + *op++ = 255; + *op++ = len; + *op++ = 255; + *op++ = distance >> 8; + *op++ = distance & 255; + } + } +#else + + if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2)) + while(len > MAX_LEN-2) + { + *op++ = (7 << 5) + (distance >> 8); + *op++ = MAX_LEN - 2 - 7 -2; + *op++ = (distance & 255); + len -= MAX_LEN-2; + } + + if(len < 7) + { + *op++ = (len << 5) + (distance >> 8); + *op++ = (distance & 255); + } + else + { + *op++ = (7 << 5) + (distance >> 8); + *op++ = len - 7; + *op++ = (distance & 255); + } +#endif + + /* update the hash at match boundary */ + HASH_FUNCTION(hval,ip); + htab[hval] = ip++; + HASH_FUNCTION(hval,ip); + htab[hval] = ip++; + + /* assuming literal copy */ + *op++ = MAX_COPY-1; + + continue; + + literal: + *op++ = *anchor++; + ip = anchor; + copy++; + if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) + { + copy = 0; + *op++ = MAX_COPY-1; + } + } + + /* left-over as literal copy */ + ip_bound++; + while(ip <= ip_bound) + { + *op++ = *ip++; + copy++; + if(copy == MAX_COPY) + { + copy = 0; + *op++ = MAX_COPY-1; + } + } + + /* if we have copied something, adjust the copy length */ + if(copy) + *(op-copy-1) = copy-1; + else + op--; + +#if FASTLZ_LEVEL==2 + /* marker for fastlz2 */ + *(flzuint8*)output |= (1 << 5); +#endif + + return op - (flzuint8*)output; +} + +static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout) +{ + const flzuint8* ip = (const flzuint8*) input; + const flzuint8* ip_limit = ip + length; + flzuint8* op = (flzuint8*) output; + flzuint8* op_limit = op + maxout; + flzuint32 ctrl = (*ip++) & 31; + int loop = 1; + + do + { + const flzuint8* ref = op; + flzuint32 len = ctrl >> 5; + flzuint32 ofs = (ctrl & 31) << 8; + + if(ctrl >= 32) + { +#if FASTLZ_LEVEL==2 + flzuint8 code; +#endif + len--; + ref -= ofs; + if (len == 7-1) +#if FASTLZ_LEVEL==1 + len += *ip++; + ref -= *ip++; +#else + do + { + code = *ip++; + len += code; + } while (code==255); + code = *ip++; + ref -= code; + + /* match from 16-bit distance */ + if(FASTLZ_UNEXPECT_CONDITIONAL(code==255)) + if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8))) + { + ofs = (*ip++) << 8; + ofs += *ip++; + ref = op - ofs - MAX_DISTANCE; + } +#endif + +#ifdef FASTLZ_SAFE + if (FASTLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) + return 0; + + if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output)) + return 0; +#endif + + if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit)) + ctrl = *ip++; + else + loop = 0; + + if(ref == op) + { + /* optimize copy for a run */ + flzuint8 b = ref[-1]; + *op++ = b; + *op++ = b; + *op++ = b; + for(; len; --len) + *op++ = b; + } + else + { +#if !defined(FASTLZ_STRICT_ALIGN) + const flzuint16* p; + flzuint16* q; +#endif + /* copy from reference */ + ref--; + *op++ = *ref++; + *op++ = *ref++; + *op++ = *ref++; + +#if !defined(FASTLZ_STRICT_ALIGN) + /* copy a byte, so that now it's word aligned */ + if(len & 1) + { + *op++ = *ref++; + len--; + } + + /* copy 16-bit at once */ + q = (flzuint16*) op; + op += len; + p = (const flzuint16*) ref; + for(len>>=1; len > 4; len-=4) + { + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + *q++ = *p++; + } + for(; len; --len) + *q++ = *p++; +#else + for(; len; --len) + *op++ = *ref++; +#endif + } + } + else + { + ctrl++; +#ifdef FASTLZ_SAFE + if (FASTLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) + return 0; + if (FASTLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) + return 0; +#endif + + *op++ = *ip++; + for(--ctrl; ctrl; ctrl--) + *op++ = *ip++; + + loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit); + if(loop) + ctrl = *ip++; + } + } + while(FASTLZ_EXPECT_CONDITIONAL(loop)); + + return op - (flzuint8*)output; +} + +#endif /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */ diff --git a/Modem/compression/fastlz.h b/Modem/compression/fastlz.h new file mode 100644 index 0000000..f87bc7b --- /dev/null +++ b/Modem/compression/fastlz.h @@ -0,0 +1,100 @@ +/* + FastLZ - lightning-fast lossless compression library + + Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) + Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) + Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#ifndef FASTLZ_H +#define FASTLZ_H + +#define FASTLZ_VERSION 0x000100 + +#define FASTLZ_VERSION_MAJOR 0 +#define FASTLZ_VERSION_MINOR 0 +#define FASTLZ_VERSION_REVISION 0 + +#define FASTLZ_VERSION_STRING "0.1.0" + +#if defined (__cplusplus) +extern "C" { +#endif + +/** + Compress a block of data in the input buffer and returns the size of + compressed block. The size of input buffer is specified by length. The + minimum input buffer size is 16. + + The output buffer must be at least 5% larger than the input buffer + and can not be smaller than 66 bytes. + + If the input is not compressible, the return value might be larger than + length (input buffer size). + + The input buffer and the output buffer can not overlap. +*/ + +int fastlz_compress(const void* input, int length, void* output); + +/** + Decompress a block of compressed data and returns the size of the + decompressed block. If error occurs, e.g. the compressed data is + corrupted or the output buffer is not large enough, then 0 (zero) + will be returned instead. + + The input buffer and the output buffer can not overlap. + + Decompression is memory safe and guaranteed not to write the output buffer + more than what is specified in maxout. + */ + +int fastlz_decompress(const void* input, int length, void* output, int maxout); + +/** + Compress a block of data in the input buffer and returns the size of + compressed block. The size of input buffer is specified by length. The + minimum input buffer size is 16. + + The output buffer must be at least 5% larger than the input buffer + and can not be smaller than 66 bytes. + + If the input is not compressible, the return value might be larger than + length (input buffer size). + + The input buffer and the output buffer can not overlap. + + Compression level can be specified in parameter level. At the moment, + only level 1 and level 2 are supported. + Level 1 is the fastest compression and generally useful for short data. + Level 2 is slightly slower but it gives better compression ratio. + + Note that the compressed data, regardless of the level, can always be + decompressed using the function fastlz_decompress above. +*/ + +int fastlz_compress_level(int level, const void* input, int length, void* output); + +#if defined (__cplusplus) +} +#endif + +#endif /* FASTLZ_H */ diff --git a/Modem/compression/lzfx.c b/Modem/compression/lzfx.c new file mode 100644 index 0000000..9bf7cd3 --- /dev/null +++ b/Modem/compression/lzfx.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2009 Andrew Collette + * http://lzfx.googlecode.com + * + * Implements an LZF-compatible compressor/decompressor based on the liblzf + * codebase written by Marc Lehmann. This code is released under the BSD + * license. License and original copyright statement follow. + * + * + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "lzfx.h" + +#define LZFX_HSIZE (1 << (LZFX_HLOG)) + +/* We need this for memset */ +# include + +// #if __GNUC__ >= 3 && !DISABLE_EXPECT +// # define fx_expect_false(expr) __builtin_expect((expr) != 0, 0) +// # define fx_expect_true(expr) __builtin_expect((expr) != 0, 1) +// #else +# define fx_expect_false(expr) (expr) +# define fx_expect_true(expr) (expr) +//#endif + +typedef unsigned char u8; +typedef const u8 *LZSTATE[LZFX_HSIZE]; + +/* Define the hash function */ +#define LZFX_FRST(p) (((p[0]) << 8) | p[1]) +#define LZFX_NEXT(v,p) (((v) << 8) | p[2]) +#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1)) + +/* These cannot be changed, as they are related to the compressed format. */ +#define LZFX_MAX_LIT (1 << 5) +#define LZFX_MAX_OFF (1 << 13) +#define LZFX_MAX_REF ((1 << 8) + (1 << 3)) + +static +int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen); + +/* Compressed format + + There are two kinds of structures in LZF/LZFX: literal runs and back + references. The length of a literal run is encoded as L - 1, as it must + contain at least one byte. Literals are encoded as follows: + + 000LLLLL + + Back references are encoded as follows. The smallest possible encoded + length value is 1, as otherwise the control byte would be recognized as + a literal run. Since at least three bytes must match for a back reference + to be inserted, the length is encoded as L - 2 instead of L - 1. The + offset (distance to the desired data in the output buffer) is encoded as + o - 1, as all offsets are at least 1. The binary format is: + + LLLooooo oooooooo for backrefs of real length < 9 (1 <= L < 7) + 111ooooo LLLLLLLL oooooooo for backrefs of real length >= 9 (L > 7) +*/ +#include +int lzfx_compress(const void *const ibuf, const unsigned int ilen, + void *obuf, unsigned int *const olen){ + + /* Hash table; an array of u8*'s which point + to various locations in the input buffer */ + const u8 *htab[LZFX_HSIZE]; + + const u8 **hslot; /* Pointer to entry in hash table */ + unsigned int hval; /* Hash value generated by macros above */ + const u8 *ref; /* Pointer to candidate match location in input */ + + const u8 *ip = (const u8 *)ibuf; + const u8 *const in_end = ip + ilen; + + u8 *op = (u8 *)obuf; + const u8 *const out_end = (olen == NULL ? NULL : op + *olen); + + int lit; /* # of bytes in current literal run */ + +#if defined (WIN32) && defined (_M_X64) + unsigned _int64 off; /* workaround for missing POSIX compliance */ +#else + unsigned long off; +#endif + + if(olen == NULL) return LZFX_EARGS; + if(ibuf == NULL){ + if(ilen != 0) return LZFX_EARGS; + *olen = 0; + return 0; + } + if(obuf == NULL){ + if(olen != 0) return LZFX_EARGS; + return lzfx_getsize(ibuf, ilen, olen); + } + + memset(htab, 0, sizeof(htab)); + + /* Start a literal run. Whenever we do this the output pointer is + advanced because the current byte will hold the encoded length. */ + lit = 0; op++; + + hval = LZFX_FRST(ip); + + while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */ + + hval = LZFX_NEXT(hval, ip); + hslot = htab + LZFX_IDX(hval); + + ref = *hslot; *hslot = ip; + + if( ref < ip + && (off = ip - ref - 1) < LZFX_MAX_OFF + && ip + 4 < in_end /* Backref takes up to 3 bytes, so don't bother */ + && ref > (u8 *)ibuf + && ref[0] == ip[0] + && ref[1] == ip[1] + && ref[2] == ip[2] ) { + + unsigned int len = 3; /* We already know 3 bytes match */ + const unsigned int maxlen = in_end - ip - 2 > LZFX_MAX_REF ? + LZFX_MAX_REF : in_end - ip - 2; + + /* lit == 0: op + 3 must be < out_end (because we undo the run) + lit != 0: op + 3 + 1 must be < out_end */ + if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) + return LZFX_ESIZE; + + op [- lit - 1] = lit - 1; /* Terminate literal run */ + op -= !lit; /* Undo run if length is zero */ + + /* Start checking at the fourth byte */ + while (len < maxlen && ref[len] == ip[len]) + len++; + + len -= 2; /* We encode the length as #octets - 2 */ + + /* Format 1: [LLLooooo oooooooo] */ + if (len < 7) { + *op++ = (off >> 8) + (len << 5); + *op++ = off; + + /* Format 2: [111ooooo LLLLLLLL oooooooo] */ + } else { + *op++ = (off >> 8) + (7 << 5); + *op++ = len - 7; + *op++ = off; + } + + lit = 0; op++; + + ip += len + 1; /* ip = initial ip + #octets -1 */ + + if (fx_expect_false (ip + 3 >= in_end)){ + ip++; /* Code following expects exit at bottom of loop */ + break; + } + + hval = LZFX_FRST (ip); + hval = LZFX_NEXT (hval, ip); + htab[LZFX_IDX (hval)] = ip; + + ip++; /* ip = initial ip + #octets */ + + } else { + /* Keep copying literal bytes */ + + if (fx_expect_false (op >= out_end)) return LZFX_ESIZE; + + lit++; *op++ = *ip++; + + if (fx_expect_false (lit == LZFX_MAX_LIT)) { + op [- lit - 1] = lit - 1; /* stop run */ + lit = 0; op++; /* start run */ + } + + } /* if() found match in htab */ + + } /* while(ip < ilen -2) */ + + /* At most 3 bytes remain in input. We therefore need 4 bytes available + in the output buffer to store them (3 data + ctrl byte).*/ + if (op + 3 > out_end) return LZFX_ESIZE; + + while (ip < in_end) { + + lit++; *op++ = *ip++; + + if (fx_expect_false (lit == LZFX_MAX_LIT)){ + op [- lit - 1] = lit - 1; + lit = 0; op++; + } + } + + op [- lit - 1] = lit - 1; + op -= !lit; + + *olen = op - (u8 *)obuf; + return 0; +} + +/* Decompressor */ +int lzfx_decompress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen){ + + u8 const *ip = (const u8 *)ibuf; + u8 const *const in_end = ip + ilen; + u8 *op = (u8 *)obuf; + u8 const *const out_end = (olen == NULL ? NULL : op + *olen); + + unsigned int remain_len = 0; + int rc; + + if(olen == NULL) return LZFX_EARGS; + if(ibuf == NULL){ + if(ilen != 0) return LZFX_EARGS; + *olen = 0; + return 0; + } + if(obuf == NULL){ + if(olen != 0) return LZFX_EARGS; + return lzfx_getsize(ibuf, ilen, olen); + } + + do { + unsigned int ctrl = *ip++; + + /* Format 000LLLLL: a literal byte string follows, of length L+1 */ + if(ctrl < (1 << 5)) { + + ctrl++; + + if(fx_expect_false(op + ctrl > out_end)){ + --ip; /* Rewind to control byte */ + goto guess; + } + if(fx_expect_false(ip + ctrl > in_end)) return LZFX_ECORRUPT; + + do + *op++ = *ip++; + while(--ctrl); + + /* Format #1 [LLLooooo oooooooo]: backref of length L+1+2 + ^^^^^ ^^^^^^^^ + A B + #2 [111ooooo LLLLLLLL oooooooo] backref of length L+7+2 + ^^^^^ ^^^^^^^^ + A B + In both cases the location of the backref is computed from the + remaining part of the data as follows: + + location = op - A*256 - B - 1 + */ + } else { + + unsigned int len = (ctrl >> 5); + u8 *ref = op - ((ctrl & 0x1f) << 8) -1; + + if(len==7) len += *ip++; /* i.e. format #2 */ + + len += 2; /* len is now #octets */ + + if(fx_expect_false(op + len > out_end)){ + ip -= (len >= 9) ? 2 : 1; /* Rewind to control byte */ + goto guess; + } + if(fx_expect_false(ip >= in_end)) return LZFX_ECORRUPT; + + ref -= *ip++; + + if(fx_expect_false(ref < (u8*)obuf)) return LZFX_ECORRUPT; + + do + *op++ = *ref++; + while (--len); + } + + } while (ip < in_end); + + *olen = op - (u8 *)obuf; + + return 0; + +guess: + rc = lzfx_getsize(ip, ilen - (ip-(u8*)ibuf), &remain_len); + if(rc>=0) *olen = remain_len + (op - (u8*)obuf); + return rc; +} + +/* Guess len. No parameters may be NULL; this is not checked. */ +static +int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen){ + + u8 const *ip = (const u8 *)ibuf; + u8 const *const in_end = ip + ilen; + int tot_len = 0; + + while (ip < in_end) { + + unsigned int ctrl = *ip++; + + if(ctrl < (1 << 5)) { + + ctrl++; + + if(ip + ctrl > in_end) + return LZFX_ECORRUPT; + + tot_len += ctrl; + ip += ctrl; + + } else { + + unsigned int len = (ctrl >> 5); + + if(len==7){ /* i.e. format #2 */ + len += *ip++; + } + + len += 2; /* len is now #octets */ + + if(ip >= in_end) return LZFX_ECORRUPT; + + ip++; /* skip the ref byte */ + + tot_len += len; + + } + + } + + *olen = tot_len; + + return 0; +} + + + + diff --git a/Modem/compression/lzfx.h b/Modem/compression/lzfx.h new file mode 100644 index 0000000..13c9e8f --- /dev/null +++ b/Modem/compression/lzfx.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2009 Andrew Collette + * http://lzfx.googlecode.com + * + * Implements an LZF-compatible compressor/decompressor based on the liblzf + * codebase written by Marc Lehmann. This code is released under the BSD + * license. License and original copyright statement follow. + * + * + * Copyright (c) 2000-2008 Marc Alexander Lehmann + * + * Redistribution and use in source and binary forms, with or without modifica- + * tion, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- + * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- + * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- + * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZFX_H +#define LZFX_H + +/* Documented behavior, including function signatures and error codes, + is guaranteed to remain unchanged for releases with the same major + version number. Releases of the same major version are also able + to read each other's output, although the output itself is not + guaranteed to be byte-for-byte identical. +*/ +#define LZFX_VERSION_MAJOR 0 +#define LZFX_VERSION_MINOR 1 +#define LZFX_VERSION_STRING "0.1" + +/* Hashtable size (2**LZFX_HLOG entries) */ +#ifndef LZFX_HLOG +# define LZFX_HLOG 8 +#endif + +/* Predefined errors. */ +#define LZFX_ESIZE -1 /* Output buffer too small */ +#define LZFX_ECORRUPT -2 /* Invalid data for decompression */ +#define LZFX_EARGS -3 /* Arguments invalid (NULL) */ + +/* Buffer-to buffer compression. + + Supply pre-allocated input and output buffers via ibuf and obuf, and + their size in bytes via ilen and olen. Buffers may not overlap. + + On success, the function returns a non-negative value and the argument + olen contains the compressed size in bytes. On failure, a negative + value is returned and olen is not modified. +*/ +int lzfx_compress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen); + +/* Buffer-to-buffer decompression. + + Supply pre-allocated input and output buffers via ibuf and obuf, and + their size in bytes via ilen and olen. Buffers may not overlap. + + On success, the function returns a non-negative value and the argument + olen contains the uncompressed size in bytes. On failure, a negative + value is returned. + + If the failure code is LZFX_ESIZE, olen contains the minimum buffer size + required to hold the decompressed data. Otherwise, olen is not modified. + + Supplying a zero *olen is a valid and supported strategy to determine the + required buffer size. This does not require decompression of the entire + stream and is consequently very fast. Argument obuf may be NULL in + this case only. +*/ +int lzfx_decompress(const void* ibuf, unsigned int ilen, + void* obuf, unsigned int *olen); + + +#endif diff --git a/Modem/config.h b/Modem/config.h index 343466f..35ff42e 100644 --- a/Modem/config.h +++ b/Modem/config.h @@ -12,6 +12,6 @@ // will wait for data before timing out. #define CONFIG_AFSK_PREAMBLE_LEN 10UL // The length of the packet preamble in milliseconds -#define CONFIG_AFSK_TRAILER_LEN 2UL // The length of the packet tail in milliseconds +#define CONFIG_AFSK_TRAILER_LEN 2UL // The length of the packet tail in milliseconds #endif \ No newline at end of file diff --git a/Modem/main.c b/Modem/main.c index 84a91c9..a933215 100644 --- a/Modem/main.c +++ b/Modem/main.c @@ -15,6 +15,7 @@ #include "afsk.h" // Header for AFSK modem #include "protocol/mp1.h" // Header for MP.1 protocol + ////////////////////////////////////////////////////// // A few definitions // ////////////////////////////////////////////////////// @@ -51,6 +52,12 @@ static void mp1Callback(struct MP1Packet *packet) { //kprintf("%.*s\n", packet->dataLength, packet->data); } +// static int freeRam () { +// extern int __heap_start, *__brkval; +// int v; +// return (int) &v - (__brkval == 0 ? (int) &__heap_start : (int) __brkval); +// } + // Simple initialization function. static void init(void) { @@ -119,6 +126,7 @@ int main(void) // If we should, pass the buffer to the protocol's // send function. mp1Send(&mp1, serialBuffer, serialLen); + // Reset the transmission flag and length counter sertx = false; serialLen = 0; diff --git a/Modem/protocol/mp1.h b/Modem/protocol/mp1.h index 6606812..a41bb95 100644 --- a/Modem/protocol/mp1.h +++ b/Modem/protocol/mp1.h @@ -4,9 +4,11 @@ #include #include +#include "compression/lzfx.h" + // Frame sizing & checksum #define MP1_MIN_FRAME_LENGTH 3 -#define MP1_MAX_FRAME_LENGTH 300 +#define MP1_MAX_FRAME_LENGTH 200 #define MP1_CHECKSUM_INIT 0xAA // We need to know some basic HDLC flag bytes diff --git a/buildrev.h b/buildrev.h index 7025c3c..ef1cde4 100644 --- a/buildrev.h +++ b/buildrev.h @@ -1,2 +1,2 @@ -#define VERS_BUILD 835 +#define VERS_BUILD 930 #define VERS_HOST "vixen"