diff --git a/inst/bdivide/compress.lua b/inst/bdivide/compress.lua index 1ffd910..6a36450 100644 --- a/inst/bdivide/compress.lua +++ b/inst/bdivide/compress.lua @@ -41,8 +41,9 @@ for i = 0, 127 do end end -local function preproc(blk, p) +local function preprocWithPadding(blk, p) local out = "" + local needsPadding = false while blk ~= "" do p(blk) local len = math.min(preprocMaxLen, #blk) @@ -50,6 +51,7 @@ local function preproc(blk, p) local seg = blk:sub(1, len) if preprocParts[seg] then out = out .. preprocParts[seg] + needsPadding = #preprocParts[seg] < 2 blk = blk:sub(#seg + 1) break end @@ -57,68 +59,27 @@ local function preproc(blk, p) end assert(len ~= 0) end - return out -end - --- BDIVIDE r5 edition --- Algorithm simplified for smaller implementation and potentially better compression --- format: --- 0-127 for constants --- <128 + (length - 4)>, , --- Position is where in the window it was found, minus 1. --- windowSize must be the same between the encoder and decoder, --- and is the amount of data preserved after cropping. -local function bdivide(blk, p) - local out = "" - - local windowSize = 0x10000 - local windowData = ("\x00"):rep(windowSize) - - while blk ~= "" do - p(blk) - local bestData = blk:sub(1, 1) - local bestRes = bestData - for lm = 0, 127 do - local al = lm + 4 - local pfx = blk:sub(1, al) - if #pfx ~= al then - break - end - local p = windowData:find(pfx, 1, true) - if not p then - break - end - local pm = p - 1 - local thirdByte = pm % 256 - -- anti ']'-corruption helper - if thirdByte ~= 93 then - bestData = string.char(128 + lm, math.floor(pm / 256), thirdByte) - bestRes = pfx - end - end - -- ok, encode! - out = out .. bestData - -- crop window - windowData = (windowData .. bestRes):sub(-windowSize) - blk = blk:sub(#bestRes + 1) + -- This needsPadding bit is just sort of quickly added in + -- to keep this part properly maintained + -- even though it might never get used + if needsPadding then + return out .. "\x00" end return out end +local bdCore = require("bdivide.core") + return function (data, lexCrunch) io.stderr:write("preproc: ") local pi = frw.progress() local function p(b) pi(1 - (#b / #data)) end - data = preproc(data, p) + data = preprocWithPadding(data, p) io.stderr:write("\nbdivide: ") pi = frw.progress() - data = bdivide(data, p) + data = bdCore.bdividePad(bdCore.bdivide(data, p)) io.stderr:write("\n") - -- These are used to pad the stream to flush the pipeline. - -- It's cheaper than the required code. - -- 1 byte of buffer for preproc, - -- 2 bytes of buffer for bdivide. - return lexCrunch.process(frw.read("bdivide/instdeco.lua"), {}), data .. ("\x00"):rep(3) + return lexCrunch.process(frw.read("bdivide/instdeco.lua"), {}), data end diff --git a/inst/bdivide/core.lua b/inst/bdivide/core.lua new file mode 100644 index 0000000..b6a18db --- /dev/null +++ b/inst/bdivide/core.lua @@ -0,0 +1,72 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +-- BDIVIDE r5 edition +-- Algorithm simplified for smaller implementation and potentially better compression +-- format: +-- 0-127 for constants +-- <128 + (length - 4)>, , +-- Position is where in the window it was found, minus 1. +-- windowSize must be the same between the encoder and decoder, +-- and is the amount of data preserved after cropping. + +local bdivCore = {} + +function bdivCore.bdivide(blk, p) + local out = "" + + local windowSize = 0x10000 + local windowData = ("\x00"):rep(windowSize) + + while blk ~= "" do + p(blk) + local bestData = blk:sub(1, 1) + assert(blk:byte() < 128, "BDIVIDE does not handle 8-bit data") + local bestRes = bestData + for lm = 0, 127 do + local al = lm + 4 + local pfx = blk:sub(1, al) + if #pfx ~= al then + break + end + local p = windowData:find(pfx, 1, true) + if not p then + break + end + local pm = p - 1 + local thirdByte = pm % 256 + bestData = string.char(128 + lm, math.floor(pm / 256), thirdByte) + bestRes = pfx + end + -- ok, encode! + out = out .. bestData + -- crop window + windowData = (windowData .. bestRes):sub(-windowSize) + blk = blk:sub(#bestRes + 1) + end + return out +end + +-- Adds padding if required +function bdivCore.bdividePad(data) + local i = 1 + -- Basically, if it ends on a literal, + -- then the literal won't get read without two padding bytes. + -- Otherwise (including if no data) it's fine. + local needsPadding = false + while i <= #data do + if data:byte(i) > 127 then + i = i + 3 + needsPadding = false + else + i = i + 1 + needsPadding = true + end + end + if needsPadding then + return data .. "\x00\x00" + end + return data +end + +return bdivCore diff --git a/inst/bdivide/instdeco.lua b/inst/bdivide/instdeco.lua index 98d660d..aef893e 100644 --- a/inst/bdivide/instdeco.lua +++ b/inst/bdivide/instdeco.lua @@ -34,29 +34,31 @@ function $bdPP(x, y) return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2 end -function $engineInput($a0) - $bdBDBuffer = $bdBDBuffer .. $a0 +${ +function $engineInput($L|lData) + $bdBDBuffer = $bdBDBuffer .. $lData while #$bdBDBuffer > 2 do - $a0 = $bdBDBuffer:byte() - if $a0 < 128 then - $a0 = $bdBDBuffer:sub(1, 1) + $lData = $bdBDBuffer:byte() + if $lData < 128 then + $lData = $bdBDBuffer:sub(1, 1) $bdBDBuffer = $bdBDBuffer:sub(2) else - $NT|bdBDPtr - $bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1 - $a0 = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $a0 - 125) + ${ + $L|bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1 + $lData = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $lData - 125) $bdBDBuffer = $bdBDBuffer:sub(4) - $DT|bdBDPtr + $} end - $bdPPBuffer = $bdPPBuffer .. $a0 - $bdBDWindow = ($bdBDWindow .. $a0):sub(-2^16) + $bdPPBuffer = $bdPPBuffer .. $lData + $bdBDWindow = ($bdBDWindow .. $lData):sub(-2^16) while #$bdPPBuffer > 1 do - $NT|bdPPAdv - $a0, $bdPPAdv = $bdPP($bdPPBuffer:byte(), $bdPPBuffer:byte(2)) + ${ + $lData, $L|bdPPAdv = $bdPP($bdPPBuffer:byte(), $bdPPBuffer:byte(2)) $bdPPBuffer = $bdPPBuffer:sub($bdPPAdv) - $DT|bdPPAdv - $engineOutput($a0) + $} + $engineOutput($lData) end end end +$} diff --git a/inst/bdvlite/compress.lua b/inst/bdvlite/compress.lua new file mode 100644 index 0000000..e6a6605 --- /dev/null +++ b/inst/bdvlite/compress.lua @@ -0,0 +1,18 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +local frw = require("libs.frw") + +local bdCore = require("bdivide.core") + +return function (data, lexCrunch) + io.stderr:write("\nbdivide: ") + local pi = frw.progress() + local function p(b) + pi(1 - (#b / #data)) + end + data = bdCore.bdividePad(bdCore.bdivide(data, p)) + io.stderr:write("\n") + return lexCrunch.process(frw.read("bdvlite/instdeco.lua"), {}), data +end + diff --git a/inst/bdvlite/core.lua b/inst/bdvlite/core.lua new file mode 100644 index 0000000..b6a18db --- /dev/null +++ b/inst/bdvlite/core.lua @@ -0,0 +1,72 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +-- BDIVIDE r5 edition +-- Algorithm simplified for smaller implementation and potentially better compression +-- format: +-- 0-127 for constants +-- <128 + (length - 4)>, , +-- Position is where in the window it was found, minus 1. +-- windowSize must be the same between the encoder and decoder, +-- and is the amount of data preserved after cropping. + +local bdivCore = {} + +function bdivCore.bdivide(blk, p) + local out = "" + + local windowSize = 0x10000 + local windowData = ("\x00"):rep(windowSize) + + while blk ~= "" do + p(blk) + local bestData = blk:sub(1, 1) + assert(blk:byte() < 128, "BDIVIDE does not handle 8-bit data") + local bestRes = bestData + for lm = 0, 127 do + local al = lm + 4 + local pfx = blk:sub(1, al) + if #pfx ~= al then + break + end + local p = windowData:find(pfx, 1, true) + if not p then + break + end + local pm = p - 1 + local thirdByte = pm % 256 + bestData = string.char(128 + lm, math.floor(pm / 256), thirdByte) + bestRes = pfx + end + -- ok, encode! + out = out .. bestData + -- crop window + windowData = (windowData .. bestRes):sub(-windowSize) + blk = blk:sub(#bestRes + 1) + end + return out +end + +-- Adds padding if required +function bdivCore.bdividePad(data) + local i = 1 + -- Basically, if it ends on a literal, + -- then the literal won't get read without two padding bytes. + -- Otherwise (including if no data) it's fine. + local needsPadding = false + while i <= #data do + if data:byte(i) > 127 then + i = i + 3 + needsPadding = false + else + i = i + 1 + needsPadding = true + end + end + if needsPadding then + return data .. "\x00\x00" + end + return data +end + +return bdivCore diff --git a/inst/bdvlite/instdeco.lua b/inst/bdvlite/instdeco.lua new file mode 100644 index 0000000..3feaee8 --- /dev/null +++ b/inst/bdvlite/instdeco.lua @@ -0,0 +1,30 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +-- BDIVIDE (r5 edition) +-- decompression engine used to decompress DEFLATE decompression engine + +$bdBDBuffer = "" +$bdBDWindow = ("\x00"):rep(2^16) + +${ +function $engineInput($L|lData) + $bdBDBuffer = $bdBDBuffer .. $lData + while #$bdBDBuffer > 2 do + $lData = $bdBDBuffer:byte() + if $lData < 128 then + $lData = $bdBDBuffer:sub(1, 1) + $bdBDBuffer = $bdBDBuffer:sub(2) + else + ${ + $L|bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1 + $lData = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $lData - 125) + $bdBDBuffer = $bdBDBuffer:sub(4) + $} + end + $bdBDWindow = ($bdBDWindow .. $lData):sub(-2^16) + $engineOutput($lData) + end +end +$} + diff --git a/inst/build.lua b/inst/build.lua index 89be38c..f1491ae 100644 --- a/inst/build.lua +++ b/inst/build.lua @@ -2,46 +2,74 @@ -- No warranty is provided, implied or otherwise. -- KittenOS NEO Installer Generator -- -local alg, tarName, cid = ... -cid = (cid or "UNKNOWN"):sub(1, 7) +local args = {...} + +local cid = args[1] +local tarName = args[2] +local algorithmsInReverseOrder = {} +for i = 3, #args do + table.insert(algorithmsInReverseOrder, 1, args[i]) +end local u = require("libs.frw") -local algImpl = require(alg .. ".compress") - local instSize = 0 local function put(data) io.write(data) instSize = instSize + #data end --- TAR File -- -local tarData = u.read(tarName) -local tarSectors = math.floor(#tarData / 512) - -- Installer Lexcrunch Context -- local lexCrunch = require("libs.lexcrunch")() -local installerCore = lexCrunch.process(u.read("instcore.lua"), {["$$SECTORS"] = tostring(tarSectors)}) -local installerHead = lexCrunch.process(u.read("insthead.lua"), {["$$CORESIZE"] = tostring(#installerCore)}) -local installerTail = lexCrunch.process(u.read("insttail.lua"), {}) +-- Installer Core -- + +-- installerFinalized: +-- Stuff that's already finished and put at the end of RISM. Prepend to this. +-- installerPayload / installerProgramLength: +-- The next-outer chunk that hasn't been written to the end of RISM +-- as the compression scheme (if one) has not been applied yet. +-- Really, installerProgramLength is only necessary because of the innermost chunk, +-- as that chunk has the TAR; additional data that's part of the same effective compression block, +-- but requires the program length to avoid it. +local installerPayload +local installerProgramLength +local installerFinalized = "" + +do + local tarData = u.read(tarName) + local tarSectors = math.floor(#tarData / 512) + local installerCore = lexCrunch.process(u.read("instcore.lua"), {["$$SECTORS"] = tostring(tarSectors)}) + installerPayload = installerCore .. tarData + installerProgramLength = #installerCore +end -- Installer Compression -- -local rawData = installerCore .. tarData -io.stderr:write("compressing...\n") -local compressionEngine, compressedData = algImpl(rawData, lexCrunch) --- RISM [[ -compressedData = compressedData:gsub("\xFE", "\xFE\xFE") -compressedData = compressedData:gsub("]]", "]\xFE]") -compressedData = "\x00" .. compressedData --- ]] -io.stderr:write("compression with " .. alg .. ": " .. #rawData .. " -> " .. #compressedData .. "\n") +for _, v in ipairs(algorithmsInReverseOrder) do + io.stderr:write("compressing (" .. v .. ")\n") + local algImpl = require(v .. ".compress") + local algEngine, algData = algImpl(installerPayload, lexCrunch) + io.stderr:write("result: " .. #installerPayload .. " -> " .. #algData .. "\n") + -- prepend the program length of the last section + algEngine = lexCrunch.process("$iBlockingLen = " .. installerProgramLength .. " " .. algEngine, {}) + -- commit + installerPayload = algEngine + installerProgramLength = #installerPayload + installerFinalized = algData .. installerFinalized +end --- Installer Final Generation -- +-- Installer Final -- + +-- This is a special case, so the program length/payload/etc. business has to be repeated. put("--" .. cid .. "\n") put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n") -put(lexCrunch.process(installerHead .. compressionEngine .. installerTail, {})) -put("--[[" .. compressedData .. "]]") +put(lexCrunch.process(u.read("insthead.lua"), {["$$CORESIZE"] = tostring(installerProgramLength)})) + +local RISM = installerPayload .. installerFinalized +RISM = RISM:gsub("\xFE", "\xFE\xFE") +RISM = RISM:gsub("]]", "]\xFE]") +RISM = "\x00" .. RISM +put("--[[" .. RISM .. "]]") -- Dumping debug info -- local dbg = io.open("iSymTab", "wb") diff --git a/inst/instcore.lua b/inst/instcore.lua index 6606499..d828934 100644 --- a/inst/instcore.lua +++ b/inst/instcore.lua @@ -28,8 +28,8 @@ $icSectorsRead = 0 $iBlockingLen = 512 function $iBlockingHook($a0) if $icBytesRemaining > 0 then - $NT|icByteAdv - $icByteAdv = math.min(512, $icBytesRemaining) + ${ + $L|icByteAdv = math.min(512, $icBytesRemaining) $icBytesRemaining = $icBytesRemaining - $icByteAdv if $icFile then $filesystem.write($icFile, $a0:sub(1, $icByteAdv)) @@ -38,7 +38,7 @@ function $iBlockingHook($a0) $icFile = nil end end - $DT|icByteAdv + $} else $icFilename = $a0:sub(1, 100):gsub("\x00", "") -- this sets up the reading/skipping of data diff --git a/inst/insthead.lua b/inst/insthead.lua index 3600706..ffb528f 100644 --- a/inst/insthead.lua +++ b/inst/insthead.lua @@ -14,21 +14,54 @@ $readInFile = $filesystem.open("init.neoi.lua", "rb") $iBlockingBuffer = "" $iBlockingLen = $$CORESIZE -$iBlockingHook = function ($a0) - -- This takes over the iBlockingHook. - assert(load($a0))() +${ +function $iBlockingHook($L|lBlock) + -- Run the next script (replacement compression engine,) + assert(load($lBlock))() end +$} -$engineOutput = function ($a0) - $iBlockingBuffer = $iBlockingBuffer .. $a0 +${ +function $engineOutput($L|lBlock) + $iBlockingBuffer = $iBlockingBuffer .. $lBlock while #$iBlockingBuffer >= $iBlockingLen do - $NT|iBlock - $iBlock = $iBlockingBuffer:sub(1, $iBlockingLen) + $lBlock = $iBlockingBuffer:sub(1, $iBlockingLen) $iBlockingBuffer = $iBlockingBuffer:sub($iBlockingLen + 1) - $iBlockingHook($iBlock) - $DT|iBlock + $iBlockingHook($lBlock) end end +$} +$engineInput = $engineOutput --- DECOMPRESSION ENGINE FOLLOWS THIS CODE -- +while true do + $readInBlock = $filesystem.read($readInFile, 1024) + ${ + for i = 1, #$readInBlock do + -- Read-in state machine + + -- IT IS VERY IMPORTANT that read-in be performed char-by-char. + -- This is because of compression chain-loading; if the switch between engines isn't "clean", + -- bad stuff happens. + + -- This character becomes invalid once + -- it gets passed to engineInput, + -- but that's the last step, so it's ok! + $L|readInChar = $readInBlock:sub(i, i) + if not $readInState then + if $readInChar == "\x00" then + $readInState = 0 + end + elseif $readInState == 0 then + if $readInChar == "\xFE" then + $readInState = 1 + else + $engineInput($readInChar) + end + else + $engineInput($readInChar) + $readInState = 0 + end + end + $} +end diff --git a/inst/insttail.lua b/inst/insttail.lua deleted file mode 100644 index 961f6e4..0000000 --- a/inst/insttail.lua +++ /dev/null @@ -1,32 +0,0 @@ --- KOSNEO installer base --- This is released into the public domain. --- No warranty is provided, implied or otherwise. - --- DECOMPRESSION ENGINE PRECEDES THIS CODE -- - -while true do - $readInBlock = $filesystem.read($readInFile, 1024) - for i = 1, #$readInBlock do - -- Read-in state machine - $NT|readInChar - $readInChar = $readInBlock:sub(i, i) - if not $readInState then - if $readInChar == "\x00" then - $readInState = 0 - end - elseif $readInState == 0 then - if $readInChar == "\xFE" then - $readInState = 1 - else - $engineInput($readInChar) - end - else - $engineInput($readInChar) - $readInState = 0 - end - end - $DT|readInChar -end - --- COMPRESSED DATA FOLLOWS THIS CODE -- - diff --git a/inst/libs/lexcrunch.lua b/inst/libs/lexcrunch.lua index f8e5cc6..d361f2b 100644 --- a/inst/libs/lexcrunch.lua +++ b/inst/libs/lexcrunch.lua @@ -135,48 +135,30 @@ return function () assert(defines[str], "no define " .. str) return defines[str] end - local comGet = str:sub(2):gmatch("[^%|]*") - local command = comGet() - if command == "NT" then - -- temporaries + - allocTmp("$" .. comGet()) - return "" - elseif command == "DT" then - -- temporaries - - local id = "$" .. comGet() - assert(forwardSymTab[id], "no such var: " .. id) - assert(reverseSymTab[forwardSymTab[id]] == "temporary", "var not allocated as temporary: " .. id) - table.insert(temporaryPool, forwardSymTab[id]) - forwardSymTab[id] = nil - return "" - elseif command == "NA" then - local id = "$" .. comGet() - local ib = "$" .. comGet() - assert(forwardSymTab[ib], "no such var: " .. ib) - assert(not forwardSymTab[id], "alias already present: " .. id) - forwardSymTab[id] = forwardSymTab[ib] - return "" - elseif command == "DA" then - local id = "$" .. comGet() - assert(forwardSymTab[id], "no entry for " .. id) - forwardSymTab[id] = nil - return "" - elseif command == "L" then - local id = "$" .. comGet() + local com = {} + for v in str:sub(2):gmatch("[^%|]*") do + table.insert(com, v) + end + if com[1] == "L" then + assert(#com == 2) + local id = "$" .. com[2] assert(stackFrames[1], "allocation of " .. id .. " outside of stack frame") table.insert(stackFrames[1], id) return allocTmp(id) - elseif command == "{" then + elseif com[1] == "{" then + assert(#com == 1) table.insert(stackFrames, 1, {}) return "" - elseif command == "}" then + elseif com[1] == "}" then + assert(#com == 1) for _, id in ipairs(table.remove(stackFrames, 1)) do table.insert(temporaryPool, forwardSymTab[id]) forwardSymTab[id] = nil end return "" else - local id = "$" .. command + assert(#com == 1) + local id = "$" .. com[1] -- normal handling if forwardSymTab[id] then return forwardSymTab[id] diff --git a/inst/symbolGuide.md b/inst/symbolGuide.md index 4bb6144..fa36123 100644 --- a/inst/symbolGuide.md +++ b/inst/symbolGuide.md @@ -1,24 +1,20 @@ # The Symbol Guide +## lexCrunch commands + The following prefixes are really special, and are lexcrunch's responsibility: "$$THING" : These are defines. "$Thing" : Writes a global into stream. If not already allocated, is allocated a global. -DEPRECATED (yes, already): - "$NT|THING" : Allocates THING from temp pool - "$DT|THING" : Returns temp THING to temp pool - -PROBABLY DEPRECATED: - "$NA|THING1|THING2" : Copies $THING2 to $THING1 in forwards table (not in backwards table) - "$DA|THING1" : Removes THING1 in forwards table. - "${" : Opens a frame. "$}" : Closes a frame. (Attached temps are released.) "$L|THING" : Allocates THING from temp pool, attaches to stack frame, writes to stream. Use inside a comment to erase the written symbol +## Conventions + The rest are convention: "$iThing" symbols are Installer Wrapper. "$icThing" symbols are Installer Core. diff --git a/laboratory/reset-i.sh b/laboratory/reset-i.sh index 740661b..5df39a3 100755 --- a/laboratory/reset-i.sh +++ b/laboratory/reset-i.sh @@ -7,5 +7,5 @@ cp ocemu.cfg.default ocemu.cfg && rm -rf c1-sda c1-sdb tmpfs mkdir c1-sda c1-sdb echo -n c1-sda > c1-eeprom/data.bin cd .. -./package.sh $1 +./package.sh $* cp inst.lua laboratory/c1-sda/init.lua diff --git a/package.sh b/package.sh index fb9d406..386e29e 100755 --- a/package.sh +++ b/package.sh @@ -15,7 +15,7 @@ cd .. # The Installer Creator cd inst -lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua +lua build.lua `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` ../code.tar $* > ../inst.lua lua status.lua ../inst.lua cd ..