Get it to work on Linux (#1)

* fix build, other stuff (i.e. asm decompressor) doesn't quite work yet

* bleh (still no worky)

* :O it works!

* ok but the rest doesn't work

* 'minor' detail

* OOPS

* _

* clean up, add wrapper script, should work now

* README formatting

* clean up python script

* ... and actually use the flags passed to the script

* fix (remove) oneKpaq output buffering in python script

* new help text in README

Author: PoroCYon

.gitignore

@@ -0,0 +1,6 @@
+obj/
+config.mk
+onekpaq
+onekpaq_encode
+onekpaq_decode
+onekpaq_context.cache*

AsmDecode.cpp

@@ -1,8 +1,14 @@
 /* Copyright (C) Teemu Suutari */
 
+#include <string.h> /* memcpy */
 #include <vector>
 #include <sys/mman.h>
+#ifndef __linux__
 #include <mach/vm_param.h>
+#else
+#include <unistd.h>
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
 
 #include "onekpaq_common.h"
 #include "AsmDecode.hpp"
@@ -42,21 +48,21 @@ static void makeRunnable(void *ptr,ulong size)
 
 std::vector<u8> AsmDecode(const std::vector<u8> &src1,const std::vector<u8> &src2,StreamCodec::EncodeMode mode,uint shift) {
 	typedef void (*AsmDecompressor)(void*,void*);
-	void *decompressors[4]={
+	static void *decompressors[4]={
 		&onekpaq_decompressor_mode1,
 		&onekpaq_decompressor_mode2,
 		&onekpaq_decompressor_mode3,
 		&onekpaq_decompressor_mode4
 	};
 
-	ulong decompressorShiftOffsets[4]={
+	static ulong decompressorShiftOffsets[4]={
 		(ulong)&onekpaq_decompressor_mode1_shift-(ulong)(void*)&onekpaq_decompressor_mode1,
 		(ulong)&onekpaq_decompressor_mode2_shift-(ulong)(void*)&onekpaq_decompressor_mode2,
 		(ulong)&onekpaq_decompressor_mode3_shift-(ulong)(void*)&onekpaq_decompressor_mode3,
 		(ulong)&onekpaq_decompressor_mode4_shift-(ulong)(void*)&onekpaq_decompressor_mode4
 	};
 
-	ulong decompressorSizes[4]={
+	static ulong decompressorSizes[4]={
 		(ulong)&onekpaq_decompressor_mode1_end-(ulong)(void*)&onekpaq_decompressor_mode1,
 		(ulong)&onekpaq_decompressor_mode2_end-(ulong)(void*)&onekpaq_decompressor_mode2,
 		(ulong)&onekpaq_decompressor_mode3_end-(ulong)(void*)&onekpaq_decompressor_mode3,
@@ -78,7 +84,14 @@ std::vector<u8> AsmDecode(const std::vector<u8> &src1,const std::vector<u8> &src
 	((u8*)(void*)decompr)[decompressorShiftOffsets[dIndex]]=shift;
 	makeRunnable((void*)decompr,decompressorSizes[dIndex]);
 
-	INFO("Running asm decompressor Xbx=%p Xdi=%p",combine.data()+src1.size(),ret.data()+destStartMargin);
+	/*fprintf(stderr, "offset = %zu, mode=%zu, dind=%zu, shift=%zu\n", src1.size(), mode, dIndex, shift);
+	for (size_t i = 0; i < ((src1.size() < 4) ? src1.size() : 4); ++i) {
+		fprintf(stderr, "src1[%zu] = 0x%x\n", i, src1[i]);
+	}
+	for (size_t i = 0; i < ((src2.size() < 4) ? src2.size() : 4); ++i) {
+		fprintf(stderr, "src2[%zu] = 0x%x\n", i, src2[i]);
+	}*/
+	//INFO("Running asm decompressor Xbx=%p Xdi=%p",combine.data()+src1.size(),ret.data()+destStartMargin);
 	auto timeTaken=Timer([&]() {
 		decompr(combine.data()+src1.size(),ret.data()+destStartMargin);
 	});

BlockCodec.cpp

@@ -546,7 +546,7 @@ void BlockCodec::CreateContextModels(const std::vector<u8> &src,bool multipleRet
 	if (src.size()<=_cm->GetStartOffset()) {
 		_models.clear();
 		_models.push_back(Model{0,2});
-		_rawLength=src.size();
+		_rawLength=uint(src.size());
 		_header=EncodeHeaderAndClean(_models);
 		_estimatedLength=uint(_header.size()+src.size())*8;
 		return;
@@ -558,7 +558,7 @@ void BlockCodec::CreateContextModels(const std::vector<u8> &src,bool multipleRet
 
 	TICK();
 	DispatchLoop(bitLength,32,[&](size_t i) {
-		_cm->CalculateAllProbabilities(probabilityMap[i],src,i+_cm->GetStartOffset()*8,_shift);
+		_cm->CalculateAllProbabilities(probabilityMap[i],src,uint(i+_cm->GetStartOffset()*8),_shift);
 	});
 
 
@@ -569,13 +569,13 @@ void BlockCodec::CreateContextModels(const std::vector<u8> &src,bool multipleRet
 	auto IterateModels=[&](std::vector<Model> &models)->std::pair<std::vector<Model>,float> {
 		// parallelization makes this a tad more complicated.
 		// first loop and then decide the best from the candidates
-		uint modelListSize=_cm->GetModelCount()*(numWeights+models.size());
+		size_t modelListSize=_cm->GetModelCount()*(numWeights+models.size());
 		std::vector<std::pair<std::vector<Model>,float>> modelList(modelListSize);
 		for (auto &it:modelList) it.second=std::numeric_limits<float>::infinity();
 
 		DispatchLoop(modelListSize,1,[&](size_t combinedIndex) {
-			uint i=combinedIndex/_cm->GetModelCount();
-			uint newModel=combinedIndex%_cm->GetModelCount();
+			uint i=uint(combinedIndex/_cm->GetModelCount());
+			uint newModel=uint(combinedIndex%_cm->GetModelCount());
 			std::vector<Model> testModels=models;
 			bool modified=false;
 			auto current=std::find_if(testModels.begin(),testModels.end(),[&](const Model &a) {

Makefile

@@ -1,47 +1,71 @@
 # Copyright (C) Teemu Suutari
 
-BITS	:= 32
+BITS	?= $(shell getconf LONG_BIT)
 VERSION	= 1.1
 
+NASM ?= nasm
+
+HAS_LIBDISPATCH ?= 0
+
 CC	= clang
 CXX	= clang++
-COMMONFLAGS = -Os -Wall -Wsign-compare -Wshorten-64-to-32 -Wno-shift-op-parentheses -DONEKPAQ_VERSION="\"$(VERSION)\""
+COMMONFLAGS = -g -Os -Wall -Wsign-compare -Wshorten-64-to-32 -Wno-shift-op-parentheses -DONEKPAQ_VERSION="\"$(VERSION)\""
+ifeq ($(BITS),32)
+COMMONFLAGS += -mtune=i386 -fno-tree-vectorize -ffloat-store -mno-sse2 -mno-mmx
+COMMONFLAGS += -mno-sse -mfpmath=387 -m80387
+endif
 CFLAGS	= $(COMMONFLAGS)
 CXXFLAGS = $(COMMONFLAGS) -std=c++14
-AFLAGS	= -O2
+AFLAGS	= -O2 -g
+
+-include config.mk
+
+ifneq ($(HAS_LIBDISPATCH),0)
+ifneq ($(LIBDISPATCH_INC_DIR),)
+COMMONFLAGS += -DHAS_LIBDISPATCH -I$(LIBDISPATCH_INC_DIR) -L$(LIBDISPATCH_LIB_DIR) -ldispatch -fblocks -lBlocksRuntime
+else
+COMMONFLAGS += -DHAS_LIBDISPATCH -ldispatch -fblocks -lBlocksRuntime
+endif
+endif
 
 # debugging...
 #COMMONFLAGS += -DDEBUG_BUILD -g
 #AFLAGS	+= -DDEBUG_BUILD -g
 
 ifeq ($(BITS),64)
 COMMONFLAGS += -m64
-AFLAGS	+= -fmacho64
+AFLAGS	+= -felf64
 else
 COMMONFLAGS += -m32
-AFLAGS	+= -fmacho32
+AFLAGS	+= -felf32
 endif
 
-PROG	= onekpaq
-SLINKS	= onekpaq_encode onekpaq_decode
-OBJS	= ArithEncoder.o ArithDecoder.o BlockCodec.o StreamCodec.o AsmDecode.o CacheFile.o \
-	onekpaq_main.o log.o \
-	$(foreach decompr,1 2 3 4,onekpaq_cfunc_$(decompr).o)
+PROG	:= onekpaq
+SLINKS	:= onekpaq_encode #onekpaq_decode
+OBJS	:= obj/ArithEncoder.cpp.o obj/ArithDecoder.cpp.o obj/BlockCodec.cpp.o obj/StreamCodec.cpp.o obj/CacheFile.cpp.o \
+	obj/onekpaq_main.cpp.o obj/log.c.o
+
+ifeq ($(BITS),32)
+OBJS := $(OBJS) obj/AsmDecode.cpp.o $(foreach decompr,1 2 3 4,obj/onekpaq_cfunc_$(decompr).asm.o)
+endif
 
 all: $(SLINKS)
 
-.asm.o:
-	nasm $(AFLAGS) $<
+%/:
+	mkdir -p "$@"
+
+#obj/%.asm.o: %.asm obj/
+#	$(NASM) $(AFLAGS) "$<" -o "$@"
 
-.cpp.o:
-	$(CXX) $(CXXFLAGS) -c $<
+obj/%.cpp.o: %.cpp obj/
+	$(CXX) $(CXXFLAGS) -c "$<" -o "$@"
 
-.c.o:
-	$(CC) $(CFLAGS) -c $<
+obj/%.c.o: %.c obj/
+	$(CC) $(CFLAGS) -c "$<" -o "$@"
 
 define decompressors
-onekpaq_cfunc_$(1).o: onekpaq_decompressor$(BITS).asm onekpaq_cfunc$(BITS).asm
-	nasm $(AFLAGS) -DONEKPAQ_DECOMPRESSOR_MODE=$(1) onekpaq_cfunc$(BITS).asm -o onekpaq_cfunc_$(1).o
+obj/onekpaq_cfunc_$(1).asm.o: onekpaq_decompressor$(BITS).asm onekpaq_cfunc$(BITS).asm obj/
+	$(NASM) $(AFLAGS) -DONEKPAQ_DECOMPRESSOR_MODE=$(1) onekpaq_cfunc$(BITS).asm -o "obj/onekpaq_cfunc_$(1).asm.o"
 endef
 
 $(foreach decompr,1 2 3 4,$(eval $(call decompressors,$(decompr))))
@@ -60,7 +84,7 @@ clean:
 define reporting
 report$(1): .PHONY
 	@rm -f tmp_out
-	@nasm -O2 -DONEKPAQ_DECOMPRESSOR_MODE=$(1) onekpaq_decompressor$(BITS).asm -o tmp_out
+	@$(NASM) -O2 -DONEKPAQ_DECOMPRESSOR_MODE=$(1) onekpaq_decompressor$(BITS).asm -o tmp_out
 	@stat -f "onekpaq decompressor mode$(1) size: %z" tmp_out
 	@rm -f tmp_out
 endef

README.md

@@ -1,3 +1,59 @@
+# oneKpaq Linux version
+
+## Dependencies
+
+You may want to install `libdispatch`, or compression will be really really
+slow. If there isn't a distro package for it, you can grab it
+[here](https://github.com/apple/swift-corelibs-libdispatch/), and then edit
+your `config.mk` to set `HAS_LIBDISPATCH` to 1, and `LIBDISPATCH_INC_DIR` and
+`LIBDISPATCH_LIB_DIR` to the right values.
+
+In any case, `libdispatch` is not a hard requirement. But you'll really really
+want it.
+
+## Compiling
+
+```sh
+$ make
+```
+
+## Running
+
+For ease of use (to compress a binary immediately into another executable
+binary), you probably want to run the Python script instead of the C++ program:
+
+```
+$ ./onekpaq.py --help
+usage: onekpaq.py [-h] [--onekpaq ONEKPAQ] [--nasm NASM] [--stub STUB]
+                  {1,2,3,4} {1,2,3} input [input ...] output
+
+positional arguments:
+  {1,2,3,4}          Selects the compression and decompression mode: '1' is 'single block,
+                     small decoder', '2' is 'multi-block, small decoder', '3' is 'single block,
+                     fast decoder', '4' is 'multi-block, fast decoder'.
+  {1,2,3}            Compression complexity to use. Higher values mean higher compression, but
+                     also longer compression times.
+  input              Input block(s)
+  output             Output executable
+
+optional arguments:
+  -h, --help         show this help message and exit
+  --onekpaq ONEKPAQ  The onekpaq_encode binary to use
+  --nasm NASM        The nasm binary to use
+  --stub STUB        The assembly decompressor stub to use
+
+$ ./onekpaq.py 3 3 hello hello.okp && ./hello.okp
+Hello world!
+```
+
+Currently, this program has to be executed in the root directory of this
+repository.
+
+-----------------
+
+## Original README
+
+```
 
                .--------. .--------.   .---.
                `-----.   |`------.  `.  \   \
@@ -6,8 +62,9 @@
                      `---'`---------'      `---'
 
 ----------------------------------------------------------------
-                     oneKpaq v1.0 by TS/TDA            
+                     oneKpaq v1.0 by TS/TDA
 ----------------------------------------------------------------
+```
 
 
 Overview
@@ -154,3 +211,4 @@ Thanks
 
 Special thanks to Firehawk - Without his help, these projects
 would have stayed at the drawing board. (Also thx for the logo)
+

SimpleDispatch.hpp

@@ -3,7 +3,7 @@
 #ifndef SIMPLEDISPATCH_H
 #define SIMPLEDISPATCH_H
 
-#ifdef __APPLE__
+#if defined(__APPLE__) || defined(HAS_LIBDISPATCH)
 
 #include <dispatch/dispatch.h>
 

StreamCodec.cpp

@@ -4,6 +4,10 @@
 #include <tuple>
 #include <string>
 #include <memory>
+#include <algorithm> /* std::reverse, std::min_element */
+#include <limits> /* std::numeric_limits */
+#include <string.h> /* memcmp */
+#include <stddef.h> /* size_t */
 
 #include "onekpaq_common.h"
 
@@ -223,7 +227,7 @@ void StreamCodec::Encode(const std::vector<std::vector<u8>> &blocks,EncodeMode m
 			// should not happen in practice. For now lets pad with 1-3 bytes
 			_destAsm1.clear();
 			_destAsm2.clear();
-			for (uint i=tmp1.size();i<4;i++) _destAsm2.push_back(0);
+			for (size_t i=tmp1.size();i<4;i++) _destAsm2.push_back(0);
 			_destAsm2.insert(_destAsm2.end(),tmp1.begin(),tmp1.end());
 			_destAsm2.insert(_destAsm2.end(),tmp2.begin(),tmp2.end());
 		} else {
@@ -281,6 +285,7 @@ void StreamCodec::LoadStream(std::vector<u8> singleStream)
 		prevEnd=headerEnd;
 		if (isSingle) break;
 	}
+	//INFO("prevEnd=%u isSingle=%u", prevEnd, isSingle);
 	_dest=std::vector<u8>(singleStream.begin()+prevEnd+(isSingle?0:2),singleStream.end());
 }
 

debug32.asm

@@ -19,7 +19,10 @@
 ; edx third param
 ; ecx fourth param
 ; return value in eax
-	[section .data]
+
+%ifndef ONEKPAQ_NO_SECTIONS
+	[section .data.onekpaq.func_r_%1]
+%endif
 %%func_r_%1:
 extern %1
 	dd %1
@@ -36,6 +39,7 @@ extern %1
 	push edi
 
 	call [dword %%func_r_%1]
+;	call %1
 
 	pop edi
 	pop esi
@@ -47,14 +51,20 @@ extern %1
 %endm
 
 %macro DEBUG_PLAIN 1
-	[section .rodata]
+%ifndef ONEKPAQ_NO_SECTIONS
+	[section .rodata.onekpaq.dbg_text]
+%endif
 %%dbg_text:
 	db %1
 	db 0
 	__SECT__
 
 	mov edi,%%dbg_text
+%ifidn __OUTPUT_FORMAT__, elf32
+	EXTCALL DebugPrint
+%else
 	EXTCALL _DebugPrint
+%endif
 %endm
 
 %macro DEBUG 1

log.c

@@ -8,8 +8,8 @@ void DebugPrint(const char *str,...)
 {
 	va_list ap;
 	va_start(ap,str);
-	vfprintf(stdout,str,ap);
-	fflush(stdout);
+	vfprintf(stderr,str,ap);
+	fflush(stderr);
 	va_end(ap);
 }