Skip to content

Commit 5237c4a

Browse files
temisutemisu
committed
import from old svn
1 parent 674cb7b commit 5237c4a

31 files changed

+2685
-0
lines changed

‎ArithDecoder.cpp

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/* Copyright (C) Teemu Suutari */
2+
3+
#include <vector>
4+
5+
#include "onekpaq_common.h"
6+
#include "ArithDecoder.hpp"
7+
8+
ArithDecoder::ArithDecoder(const std::vector<u8> &src,DecoderType type) :
9+
_type(type),
10+
_src(src)
11+
{
12+
ASSERT(_type<=DecoderType::TypeLast,"Unknown decoder type");
13+
Normalize();
14+
}
15+
16+
void ArithDecoder::Normalize()
17+
{
18+
auto decodeBit=[&]()->bool {
19+
bool ret=_srcPos>>3<_src.size()&&_src[_srcPos>>3]&0x80U>>(_srcPos&7);
20+
_srcPos++;
21+
return ret;
22+
};
23+
24+
while (_range<0x4000'0000U) {
25+
_range<<=1;
26+
_value<<=1;
27+
if (_type!=DecoderType::Standard) {
28+
if (!_srcPos) ASSERT(!decodeBit(),"Wrong start bit");
29+
if (_srcPos==6) ASSERT(decodeBit(),"Wrong anchor bit");
30+
if (_srcPos==7) ASSERT(!decodeBit(),"Wrong filler bit");
31+
}
32+
if (decodeBit()) _value++;
33+
}
34+
}
35+
36+
void ArithDecoder::ProcessEndOfSection(u32 subRange)
37+
{
38+
DEBUG("range %x, subrange %x, value %x",_range,subRange,_value);
39+
if (_type==DecoderType::SingleAsm) {
40+
ASSERT(subRange&&subRange<_range-1,"probability error");
41+
42+
_range-=subRange;
43+
ASSERT(_value==_range,"End of section not detected");
44+
_range=1;
45+
_value=0;
46+
Normalize();
47+
}
48+
DEBUG("%u bits decoded",_srcPos);
49+
}
50+
51+
void ArithDecoder::PreDecode(u32 subRange)
52+
{
53+
if (_type!=DecoderType::Standard&&!_hasPreBit) {
54+
ASSERT(!Decode(subRange),"Wrong purge bit");
55+
_hasPreBit=true;
56+
}
57+
}
58+
59+
bool ArithDecoder::Decode(u32 subRange)
60+
{
61+
DEBUG("range %x, subrange %x, value %x",_range,subRange,_value);
62+
ASSERT(subRange&&subRange<_range-1,"probability error");
63+
64+
bool ret;
65+
if (_type==DecoderType::SingleAsm) {
66+
_range-=subRange;
67+
ASSERT(_value!=_range,"End of section detected in middle of stream");
68+
ASSERT(!(_destPos&7)||_value!=_range+1,"Encoder encoded a symbol we bug on");
69+
if (_value>=_range) {
70+
_value-=_range+1;
71+
// _range=subRange-1 would be correct here.
72+
_range=subRange;
73+
ret=false;
74+
} else ret=true;
75+
} else {
76+
_range-=subRange;
77+
if (_value>=_range) {
78+
_value-=_range;
79+
_range=subRange;
80+
ret=false;
81+
} else ret=true;
82+
}
83+
84+
Normalize();
85+
_destPos++;
86+
return ret;
87+
}

‎ArithDecoder.hpp

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/* Copyright (C) Teemu Suutari */
2+
3+
#ifndef ARITHDECODER_H
4+
#define ARITHDECODER_H
5+
6+
#include <vector>
7+
8+
#include "onekpaq_common.h"
9+
10+
class ArithDecoder
11+
{
12+
public:
13+
enum class DecoderType {
14+
SingleAsm=0,
15+
MultiAsm,
16+
Standard,
17+
TypeLast=Standard
18+
};
19+
20+
ArithDecoder(const std::vector<u8> &src,DecoderType type);
21+
~ArithDecoder() = default;
22+
23+
u32 GetRange() const { return _range; }
24+
void ProcessEndOfSection(u32 subRange);
25+
void PreDecode(u32 subRange);
26+
bool Decode(u32 subRange);
27+
28+
private:
29+
void Normalize();
30+
31+
DecoderType _type;
32+
const std::vector<u8>& _src;
33+
uint _srcPos=0;
34+
uint _destPos=0;
35+
u32 _value=0;
36+
u32 _range=1;
37+
bool _hasPreBit=false;
38+
};
39+
40+
#endif

‎ArithEncoder.cpp

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/* Copyright (C) Teemu Suutari */
2+
3+
#include <vector>
4+
5+
#include "onekpaq_common.h"
6+
#include "ArithEncoder.hpp"
7+
8+
ArithEncoder::ArithEncoder(EncoderType type) :
9+
_type(type)
10+
{
11+
ASSERT(_type<=EncoderType::TypeLast,"Unknown encoder type");
12+
}
13+
14+
void ArithEncoder::Normalize()
15+
{
16+
auto encodeBit=[&](bool bit) {
17+
_dest.resize((_destPos>>3)+1);
18+
if (bit) _dest[_destPos>>3]|=0x80U>>(_destPos&7);
19+
_destPos++;
20+
};
21+
22+
while (_range<0x4000'0000U) {
23+
if (_bitCount) _bitCount--;
24+
else {
25+
if (_type!=EncoderType::Standard) {
26+
if (!_destPos) encodeBit(false); // start bit
27+
if (_destPos==6) encodeBit(true); // anchor bit
28+
if (_destPos==7) encodeBit(false); // filler bit
29+
}
30+
ASSERT(!(_value&0x8000'0000'0000'0000ULL),"Overflow");
31+
encodeBit(_value&0x4000'0000'0000'0000ULL);
32+
_value&=~0x4000'0000'0000'0000ULL;
33+
}
34+
_value<<=1;
35+
_range<<=1;
36+
}
37+
}
38+
39+
void ArithEncoder::PreEncode(u32 subRange)
40+
{
41+
if (_type!=EncoderType::Standard&&!_hasPreBit) {
42+
Encode(false,subRange);
43+
_hasPreBit=true;
44+
}
45+
}
46+
47+
void ArithEncoder::Encode(bool bit,u32 subRange)
48+
{
49+
DEBUG("bit %u, range %x, subrange %x, value %llx",bit,_range,subRange,_value);
50+
ASSERT(subRange&&subRange<_range-1,"probability error");
51+
52+
if (_type==EncoderType::SingleAsm) {
53+
if (bit) _range-=subRange;
54+
else {
55+
_value+=_range-subRange+1;
56+
// _range=subRange-1 would be correct here.
57+
_range=subRange;
58+
}
59+
} else {
60+
if (bit) _range-=subRange;
61+
else {
62+
_value+=_range-subRange;
63+
_range=subRange;
64+
}
65+
}
66+
Normalize();
67+
}
68+
69+
void ArithEncoder::EndSection(u32 subRange)
70+
{
71+
DEBUG("range %x, subrange %x, value %llx",_range,subRange,_value);
72+
ASSERT(subRange&&subRange<_range-1,"probability error at the end");
73+
74+
if (_type==EncoderType::SingleAsm) {
75+
_value+=_range-subRange;
76+
_range=1;
77+
Normalize();
78+
}
79+
}
80+
81+
void ArithEncoder::Finalize()
82+
{
83+
#if 1
84+
for (uint i=0;i<63;i++) {
85+
_range>>=1;
86+
Normalize();
87+
}
88+
#else
89+
u64 valueX=_value+_range^_value;
90+
while (!(valueX&0x4000'0000'0000'0000ULL)) {
91+
_range>>=1;
92+
Normalize();
93+
valueX<<=1;
94+
}
95+
_value|=0x4000'0000'0000'0000ULL;
96+
_range>>=1;
97+
Normalize();
98+
#endif
99+
DEBUG("%u bits encoded",_destPos);
100+
}

‎ArithEncoder.hpp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/* Copyright (C) Teemu Suutari */
2+
3+
#ifndef ARITHENCODER_H
4+
#define ARITHENCODER_H
5+
6+
#include <vector>
7+
8+
#include "onekpaq_common.h"
9+
10+
class ArithEncoder
11+
{
12+
public:
13+
enum class EncoderType {
14+
SingleAsm=0,
15+
MultiAsm,
16+
Standard,
17+
TypeLast=Standard
18+
};
19+
20+
ArithEncoder(EncoderType type);
21+
~ArithEncoder() = default;
22+
23+
u32 GetRange() const { return _range; }
24+
void PreEncode(u32 subRange);
25+
void Encode(bool bit,u32 subRange);
26+
27+
void EndSection(u32 subRange);
28+
void Finalize();
29+
30+
const std::vector<u8> &GetDest() const { return _dest; }
31+
32+
private:
33+
void Normalize();
34+
35+
EncoderType _type;
36+
std::vector<u8> _dest;
37+
uint _destPos=0;
38+
u32 _range=0x4000'0000U;
39+
u64 _value=0;
40+
uint _bitCount=33;
41+
bool _hasPreBit=false;
42+
};
43+
44+
#endif

‎AsmDecode.cpp

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/* Copyright (C) Teemu Suutari */
2+
3+
#include <vector>
4+
#include <sys/mman.h>
5+
#include <mach/vm_param.h>
6+
7+
#include "onekpaq_common.h"
8+
#include "AsmDecode.hpp"
9+
#include "Timer.hpp"
10+
11+
extern "C" void *onekpaq_decompressor_mode1;
12+
extern "C" void *onekpaq_decompressor_mode2;
13+
extern "C" void *onekpaq_decompressor_mode3;
14+
extern "C" void *onekpaq_decompressor_mode4;
15+
16+
extern "C" void *onekpaq_decompressor_mode1_shift;
17+
extern "C" void *onekpaq_decompressor_mode2_shift;
18+
extern "C" void *onekpaq_decompressor_mode3_shift;
19+
extern "C" void *onekpaq_decompressor_mode4_shift;
20+
21+
extern "C" void *onekpaq_decompressor_mode1_end;
22+
extern "C" void *onekpaq_decompressor_mode2_end;
23+
extern "C" void *onekpaq_decompressor_mode3_end;
24+
extern "C" void *onekpaq_decompressor_mode4_end;
25+
26+
// Please be PIC!
27+
static void *makeCodeblock(const void *prog,ulong size)
28+
{
29+
// not executable yet!
30+
void *ret=mmap(nullptr,(size+PAGE_SIZE-1)&~(PAGE_SIZE-1),PROT_READ|PROT_WRITE,MAP_ANON|MAP_PRIVATE,-1,0);
31+
ASSERT(ret!=MAP_FAILED,"mmap failed");
32+
memcpy(ret,prog,size);
33+
return ret;
34+
}
35+
36+
37+
static void makeRunnable(void *ptr,ulong size)
38+
{
39+
mprotect(ptr,(size+PAGE_SIZE-1)&~(PAGE_SIZE-1),PROT_READ|PROT_EXEC);
40+
}
41+
42+
43+
std::vector<u8> AsmDecode(const std::vector<u8> &src1,const std::vector<u8> &src2,StreamCodec::EncodeMode mode,uint shift) {
44+
typedef void (*AsmDecompressor)(void*,void*);
45+
void *decompressors[4]={
46+
&onekpaq_decompressor_mode1,
47+
&onekpaq_decompressor_mode2,
48+
&onekpaq_decompressor_mode3,
49+
&onekpaq_decompressor_mode4
50+
};
51+
52+
ulong decompressorShiftOffsets[4]={
53+
(ulong)&onekpaq_decompressor_mode1_shift-(ulong)(void*)&onekpaq_decompressor_mode1,
54+
(ulong)&onekpaq_decompressor_mode2_shift-(ulong)(void*)&onekpaq_decompressor_mode2,
55+
(ulong)&onekpaq_decompressor_mode3_shift-(ulong)(void*)&onekpaq_decompressor_mode3,
56+
(ulong)&onekpaq_decompressor_mode4_shift-(ulong)(void*)&onekpaq_decompressor_mode4
57+
};
58+
59+
ulong decompressorSizes[4]={
60+
(ulong)&onekpaq_decompressor_mode1_end-(ulong)(void*)&onekpaq_decompressor_mode1,
61+
(ulong)&onekpaq_decompressor_mode2_end-(ulong)(void*)&onekpaq_decompressor_mode2,
62+
(ulong)&onekpaq_decompressor_mode3_end-(ulong)(void*)&onekpaq_decompressor_mode3,
63+
(ulong)&onekpaq_decompressor_mode4_end-(ulong)(void*)&onekpaq_decompressor_mode4
64+
};
65+
66+
// we have no way of calculating the real size for dest without decompressing.
67+
// lets just put some huge number here
68+
uint expectedSize=262144;
69+
70+
std::vector<u8> ret(expectedSize);
71+
std::vector<u8> combine=src1;
72+
combine.insert(combine.end(),src2.begin(),src2.end());
73+
for (int i=0;i<4;i++) combine.push_back(0);
74+
const uint destStartMargin=9;
75+
76+
uint dIndex=static_cast<uint>(mode)-1;
77+
AsmDecompressor decompr=(AsmDecompressor)makeCodeblock(decompressors[dIndex],decompressorSizes[dIndex]);
78+
((u8*)(void*)decompr)[decompressorShiftOffsets[dIndex]]=shift;
79+
makeRunnable((void*)decompr,decompressorSizes[dIndex]);
80+
81+
INFO("Running asm decompressor Xbx=%p Xdi=%p",combine.data()+src1.size(),ret.data()+destStartMargin);
82+
auto timeTaken=Timer([&]() {
83+
decompr(combine.data()+src1.size(),ret.data()+destStartMargin);
84+
});
85+
INFO("Asm decompressor done. Time taken %f seconds",float(timeTaken));
86+
87+
return std::vector<u8>(ret.begin()+destStartMargin,ret.begin()+expectedSize-destStartMargin);
88+
}

‎AsmDecode.hpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/* Copyright (C) Teemu Suutari */
2+
3+
#ifndef ASMDECODE_H
4+
#define ASMDECODE_H
5+
6+
#include <vector>
7+
8+
#include "onekpaq_common.h"
9+
10+
#include "StreamCodec.hpp"
11+
12+
std::vector<u8> AsmDecode(const std::vector<u8> &src1,const std::vector<u8> &src2,StreamCodec::EncodeMode mode,uint shift);
13+
14+
#endif

0 commit comments

Comments
 (0)