From bc4d626b4e262e4bea3913d56402730c5d8b62d7 Mon Sep 17 00:00:00 2001 From: 20kdc Date: Tue, 31 Mar 2020 11:59:58 +0100 Subject: [PATCH] All the architectural changes pre-DEFLATE --- inst/bdivide/compress.lua | 4 +- inst/bdivide/instdeco.lua | 66 ++++++---------- inst/build.lua | 64 +++++---------- inst/instbase.lua | 139 --------------------------------- inst/instcore.lua | 76 ++++++++++++++++++ inst/insthead.lua | 34 ++++++++ inst/insttail.lua | 32 ++++++++ inst/libs/lexcrunch.lua | 86 ++++++++++++++++---- inst/status.lua | 36 +++++++++ inst/uncompressed/compress.lua | 7 +- inst/uncompressed/instdeco.lua | 13 --- inst/verify.lua | 30 ------- package.sh | 2 +- 13 files changed, 303 insertions(+), 286 deletions(-) delete mode 100644 inst/instbase.lua create mode 100644 inst/instcore.lua create mode 100644 inst/insthead.lua create mode 100644 inst/insttail.lua create mode 100644 inst/status.lua delete mode 100644 inst/uncompressed/instdeco.lua delete mode 100644 inst/verify.lua diff --git a/inst/bdivide/compress.lua b/inst/bdivide/compress.lua index 6f74477..9e6b683 100644 --- a/inst/bdivide/compress.lua +++ b/inst/bdivide/compress.lua @@ -105,7 +105,7 @@ local function bdivide(blk, p) return out end -return function (data) +return function (data, lexCrunch) io.stderr:write("preproc: ") local pi = frw.progress() local function p(b) @@ -120,5 +120,5 @@ return function (data) -- It's cheaper than the required code. -- 1 byte of buffer for preproc, -- 2 bytes of buffer for bdivide. - return data .. ("\x00"):rep(3) + return lexCrunch(frw.read("bdivide/instdeco.lua"), {}), data .. ("\x00"):rep(3) end diff --git a/inst/bdivide/instdeco.lua b/inst/bdivide/instdeco.lua index 60cf99a..b533c0d 100644 --- a/inst/bdivide/instdeco.lua +++ b/inst/bdivide/instdeco.lua @@ -4,23 +4,11 @@ -- BDIVIDE (r5 edition) and PREPROC (r9 edition) -- decompression engine for installer --- a: temporary --- touched by q,L --- b: sector accumulator --- c: bdivide accumulator --- d: temporary --- touched by q,L --- t: preproc accumulator --- q: function to submit to preproc --- s: temporary --- touched by L --- w: bdivide window - --- L: function to submit to bdivide - -b,t,c,w="","","",("\x00"):rep(2^16) +$bdPPBuffer = "" +$bdBDBuffer = "" +$bdBDWindow = ("\x00"):rep(2^16) -- High-level breakdown: --- q is unescaper & TAR-sector-breakup. +-- q is unescaper. -- It'll only begin to input if at least 3 bytes are available, -- so you'll want to throw in 2 extra zeroes at the end of stream as done here. -- It uses t (input buffer) and p (output buffer). @@ -31,7 +19,7 @@ b,t,c,w="","","",("\x00"):rep(2^16) -- And it also uses a fake local. -- SEE compress.lua FOR THIS FUNCTION -function p(x, y) +function $bdPP(x, y) if x == 126 then return string.char(y), 3 elseif x == 127 then @@ -46,31 +34,29 @@ function p(x, y) return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2 end -function q(w) - t = t .. w - while #t > 1 do - d, a = p(t:byte(), t:byte(2)) - b = b .. d - t = t:sub(a) - if #b > 511 then - M(b:sub(1, 512)) - b = b:sub(513) - end - end -end - -function L(d) - c = c .. d - while #c > 2 do - s = c:byte() - if s < 128 then - s, c = c:sub(1, 1), c:sub(2) +function $engineInput($a0) + $bdBDBuffer = $bdBDBuffer .. $a0 + while #$bdBDBuffer > 2 do + $a0 = $bdBDBuffer:byte() + if $a0 < 128 then + $a0 = $bdBDBuffer:sub(1, 1) + $bdBDBuffer = $bdBDBuffer:sub(2) else - a = c:byte(2) * 256 + c:byte(3) + 1 - s, c = w:sub(a, a + s - 125), c:sub(4) + $NTbdBDPtr + $bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1 + $a0 = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $a0 - 125) + $bdBDBuffer = $bdBDBuffer:sub(4) + $DTbdBDPtr + end + $bdPPBuffer = $bdPPBuffer .. $a0 + $bdBDWindow = ($bdBDWindow .. $a0):sub(-2^16) + while #$bdPPBuffer > 1 do + $NTbdPPAdv + $a0, $bdPPAdv = $bdPP($bdPPBuffer:byte(), $bdPPBuffer:byte(2)) + $bdPPBuffer = $bdPPBuffer:sub($bdPPAdv) + $DTbdPPAdv + $engineOutput($a0) end - q(s) - w = (w .. s):sub(-2^16) end end diff --git a/inst/build.lua b/inst/build.lua index de464f7..81d8975 100644 --- a/inst/build.lua +++ b/inst/build.lua @@ -7,8 +7,7 @@ cid = (cid or "UNKNOWN"):sub(1, 7) local u = require("libs.frw") -local tarData = u.read(tarName) -local tarSectors = math.floor(#tarData / 512) +local algImpl = require(alg .. ".compress") local instSize = 0 local function put(data) @@ -16,52 +15,31 @@ local function put(data) instSize = instSize + #data end -put("--" .. cid .. "\n") -put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n") +-- TAR File -- +local tarData = u.read(tarName) +local tarSectors = math.floor(#tarData / 512) -local instCode = "K=" .. tarSectors .. "\n" .. u.read(alg .. "/instdeco.lua") .. u.read("instbase.lua") -instCode = require("libs.lexcrunch")(instCode) -put(instCode) +-- Installer Lexcrunch Context -- +local lexCrunch = require("libs.lexcrunch")() --- the \x00 is the indicator to start reading -put("--[[\x00") +local installerCore = lexCrunch(u.read("instcore.lua"), {["$$SECTORS"] = tostring(tarSectors)}) +local installerHead = lexCrunch(u.read("insthead.lua"), {["$$CORESIZE"] = tostring(#installerCore)}) +local installerTail = lexCrunch(u.read("insttail.lua"), {}) +-- Installer Compression -- +local rawData = installerCore .. tarData io.stderr:write("compressing...\n") -local compressedData = require(alg .. ".compress")(tarData) -u.write(alg .. "/output.bin", compressedData) -io.stderr:write("compression with " .. alg .. ": " .. #tarData .. " -> " .. #compressedData .. "\n") --- Program the read-in state machine +local compressionEngine, compressedData = algImpl(rawData, lexCrunch) +-- RISM [[ compressedData = compressedData:gsub("\xFE", "\xFE\xFE") compressedData = compressedData:gsub("]]", "]\xFE]") +compressedData = "\x00" .. compressedData +-- ]] +io.stderr:write("compression with " .. alg .. ": " .. #rawData .. " -> " .. #compressedData .. "\n") -put(compressedData) -put("]]") - -local status = "" -local statusDetail = "" -local blinkI = "" -if instSize > 65536 then - blinkI = "5;31;" - status = " DO NOT SHIP " - statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code." -elseif instSize > 64000 then - blinkI = "33;" - status = " Shippable * " - statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes." -else - blinkI = "32;" - status = " All Green " - statusDetail = "The installer is well within budget with room for features.\nDevelop as normal." -end -io.stderr:write("\n") -local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n" -io.stderr:write(ctS .. " " .. ctM .. " " .. ctE) -io.stderr:write(ctS .. status .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE) -io.stderr:write(ctS .. " " .. ctM .. " " .. ctE) -io.stderr:write("\n") -io.stderr:write(statusDetail .. "\n") -io.stderr:write("\n") -io.stderr:write("Size: " .. instSize .. "\n") -io.stderr:write(" max. 65536\n") -io.stderr:write("\n") +-- Installer Final Generation -- +put("--" .. cid .. "\n") +put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n") +put(lexCrunch(installerHead .. compressionEngine .. installerTail, {})) +put("--[[" .. compressedData .. "]]") diff --git a/inst/instbase.lua b/inst/instbase.lua deleted file mode 100644 index 1c61180..0000000 --- a/inst/instbase.lua +++ /dev/null @@ -1,139 +0,0 @@ --- KOSNEO installer base --- This is released into the public domain. --- No warranty is provided, implied or otherwise. - --- DECOMPRESSION ENGINE PRECEDES THIS CODE -- --- NOTE: upper-case is reserved for this file, --- lower-case is reserved for the decompression engine - --- A: temporary - --- B: computer --- C: component - --- D: additional temporary --- E: read-in state machine - --- F: current file: filename --- H: remaining bytes to copy/skip --- I: current file: handle (nil if not writing) - --- J: sectors handled --- K: sector count (injected during build) --- L: compression engine data function (set by CE) --- M: sector handler function (called by CE) - --- O: current character for read-in state machine --- P: file handle for selfread - --- Q: octal decoding function - --- X: screen address --- Y: component: gpu --- Z: component: filesystem -B = computer -C = component -assert(C, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.") - -X = C.list("screen", true)() -Y = C.list("gpu", true)() - -Z = C.proxy(B.getBootAddress()) - -Z.remove("init.neoi.lua") -Z.rename("init.lua","init.neoi.lua") -P = Z.open("init.neoi.lua","rb") - -F = "Starting..." -H = 0 - -if X and Y then - Y = C.proxy(Y) - Y.bind(X) - Y.setResolution(50, 5) - Y.setBackground(2^24-1) - Y.setForeground(0) - Y.fill(1, 1, 50, 5, "█") - Y.fill(1, 2, 50, 1, " ") - Y.set(2, 2, "KittenOS NEO Installer") -end - -function Q(A) - if A == "" then return 0 end - return Q(A:sub(1, -2)) * 8 + (A:byte(#A) - 48) -end - -J = 0 - -function M(n) - if H > 0 then - A = math.min(512, H) - H = H - A - if I then - Z.write(I, n:sub(1, A)) - if H <= 0 then - Z.close(I) - I = nil - end - end - else - F = n:sub(1, 100):gsub("\x00", "") - -- this sets up the reading/skipping of data - H = Q(n:sub(125, 135)) - if F:sub(1, 2) == "./" and F ~= "./" then - F = F:sub(3) - if F:sub(#F) == "/" then - Z.makeDirectory(F) - else - I = Z.open(F, "wb") - if H == 0 then - Z.close(I) - I = nil - end - end - end - end - -- UPDATE DISPLAY -- - J = J + 1 - if X and Y then - Y.fill(1, 2, 50, 1, " ") - Y.set(2, 2, "KittenOS NEO Installer : " .. F) - Y.fill(2, 4, 48, 1, "█") - Y.fill(2, 4, math.ceil(48 * J / K), 1, " ") - end - if J % 8 == 0 then - B.pullSignal(0.01) - end - if J == K then - Z.close(P) - Z.remove("init.neoi.lua") - B.shutdown(true) - end -end - -while true do - A = Z.read(P, 64) - D = "" - for i = 1, #A do - -- Read-in state machine - O = A:sub(i, i) - if not E then - if O == "\x00" then - E = 0 - end - elseif E == 0 then - if O == "\xFE" then - E = 1 - else - D = D .. O - end - else - D = D .. O - E = 0 - end - end - L(D) -end - --- COMPRESSED DATA FOLLOWS THIS CODE -- - diff --git a/inst/instcore.lua b/inst/instcore.lua new file mode 100644 index 0000000..6a1ec34 --- /dev/null +++ b/inst/instcore.lua @@ -0,0 +1,76 @@ +-- KOSNEO installer base +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +$icScreen = $component.list("screen", true)() +$icGPU = $component.list("gpu", true)() + +$icFilename = "Starting..." +$icBytesRemaining = 0 + +if $icScreen and $icGPU then + $icGPU = $component.proxy($icGPU) + $icGPU.bind($icScreen) + $icGPU.setResolution(50, 5) + $icGPU.setBackground(2^24-1) + $icGPU.setForeground(0) + $icGPU.fill(1, 1, 50, 5, "█") + $icGPU.fill(1, 2, 50, 1, " ") + $icGPU.set(2, 2, "KittenOS NEO Installer") +end + +function $icOctalToNumber($a0) + if $a0 == "" then return 0 end + return $icOctalToNumber($a0:sub(1, -2)) * 8 + ($a0:byte(#$a0) - 48) +end + +$icSectorsRead = 0 +$iBlockingLen = 512 +function $iBlockingHook($a0) + if $icBytesRemaining > 0 then + $NTicByteAdv + $icByteAdv = math.min(512, $icBytesRemaining) + $icBytesRemaining = $icBytesRemaining - $icByteAdv + if $icFile then + $filesystem.write($icFile, $a0:sub(1, $icByteAdv)) + if $icBytesRemaining <= 0 then + $filesystem.close($icFile) + $icFile = nil + end + end + $DTicByteAdv + else + $icFilename = $a0:sub(1, 100):gsub("\x00", "") + -- this sets up the reading/skipping of data + $icBytesRemaining = $icOctalToNumber($a0:sub(125, 135)) + if $icFilename:sub(1, 2) == "./" and $icFilename ~= "./" then + $icFilename = $icFilename:sub(3) + if $icFilename:sub(#$icFilename) == "/" then + $filesystem.makeDirectory($icFilename) + else + $icFile = $filesystem.open($icFilename, "wb") + if $icBytesRemaining == 0 then + $filesystem.close($icFile) + $icFile = nil + end + end + end + end + -- UPDATE DISPLAY -- + $icSectorsRead = $icSectorsRead + 1 + if $icScreen and $icGPU then + $icGPU.fill(1, 2, 50, 1, " ") + $icGPU.set(2, 2, "KittenOS NEO Installer : " .. $icFilename) + $icGPU.fill(2, 4, 48, 1, "█") + $icGPU.fill(2, 4, math.ceil(48 * $icSectorsRead / $$SECTORS), 1, " ") + end + if $icSectorsRead % 16 == 0 then + $computer.pullSignal(0.01) + end + if $icSectorsRead == $$SECTORS then + $filesystem.close($readInFile) + $filesystem.remove("init.neoi.lua") + $computer.shutdown(true) + end +end + diff --git a/inst/insthead.lua b/inst/insthead.lua new file mode 100644 index 0000000..aaeee36 --- /dev/null +++ b/inst/insthead.lua @@ -0,0 +1,34 @@ +-- KOSNEO installer base +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +$computer = computer +$component = component +assert($component, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.") + +$filesystem = $component.proxy($computer.getBootAddress()) + +$filesystem.remove("init.neoi.lua") +$filesystem.rename("init.lua", "init.neoi.lua") +$readInFile = $filesystem.open("init.neoi.lua", "rb") + +$iBlockingBuffer = "" +$iBlockingLen = $$CORESIZE +$iBlockingHook = function ($a0) + -- This takes over the iBlockingHook. + assert(load($a0))() +end + +$engineOutput = function ($a0) + $iBlockingBuffer = $iBlockingBuffer .. $a0 + while #$iBlockingBuffer >= $iBlockingLen do + $NTiBlock + $iBlock = $iBlockingBuffer:sub(1, $iBlockingLen) + $iBlockingBuffer = $iBlockingBuffer:sub($iBlockingLen + 1) + $iBlockingHook($iBlock) + $DTiBlock + end +end + +-- DECOMPRESSION ENGINE FOLLOWS THIS CODE -- + diff --git a/inst/insttail.lua b/inst/insttail.lua new file mode 100644 index 0000000..8dfaac2 --- /dev/null +++ b/inst/insttail.lua @@ -0,0 +1,32 @@ +-- KOSNEO installer base +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +-- DECOMPRESSION ENGINE PRECEDES THIS CODE -- + +while true do + $readInBlock = $filesystem.read($readInFile, 1024) + for i = 1, #$readInBlock do + -- Read-in state machine + $NTreadInChar + $readInChar = $readInBlock:sub(i, i) + if not $readInState then + if $readInChar == "\x00" then + $readInState = 0 + end + elseif $readInState == 0 then + if $readInChar == "\xFE" then + $readInState = 1 + else + $engineInput($readInChar) + end + else + $engineInput($readInChar) + $readInState = 0 + end + end + $DTreadInChar +end + +-- COMPRESSED DATA FOLLOWS THIS CODE -- + diff --git a/inst/libs/lexcrunch.lua b/inst/libs/lexcrunch.lua index 0d900d6..283777f 100644 --- a/inst/libs/lexcrunch.lua +++ b/inst/libs/lexcrunch.lua @@ -72,23 +72,77 @@ local function pass(buffer) return ob end -return function (op) - -- comment removal - while true do - local np = op:gsub("%-%-[^\n]*\n", " ") - np = np:gsub("%-%-[^\n]*$", "") - if np == op then - break - end - op = np +-- Context creation -- +return function () + local forwardSymTab = {} + local reverseSymTab = {} + + local temporaryPool = {} + + local possible = {} + for i = 1, 52 do + possible[i] = ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"):sub(i, i) end - -- stripping - while true do - local np = pass(op) - if np == op then - return np + + local function allocate(reason) + for _, v in pairs(possible) do + if not reverseSymTab[v] then + reverseSymTab[v] = reason + return v + end end - op = np end - return op + + return function (op, defines) + -- symbol replacement + op = op:gsub("%$[%$a-zA-Z0-9]*", function (str) + if str:sub(2, 2) == "$" then + -- defines + assert(defines[str], "no define " .. str) + return defines[str] + elseif str:sub(2, 3) == "NT" then + -- temporaries + + local id = "$" .. str:sub(4) + assert(not forwardSymTab[id], "var already exists: " .. id) + local val = table.remove(temporaryPool, 1) + if not val then val = allocate("temporary") end + forwardSymTab[id] = val + return "" + elseif str:sub(2, 3) == "DT" then + -- temporaries - + local id = "$" .. str:sub(4) + assert(forwardSymTab[id], "no such var: " .. id) + assert(reverseSymTab[forwardSymTab[id]] == "temporary", "var not allocated as temporary: " .. id) + table.insert(temporaryPool, forwardSymTab[id]) + forwardSymTab[id] = nil + return "" + else + -- normal handling + if forwardSymTab[str] then + return forwardSymTab[str] + end + local v = allocate(str) + forwardSymTab[str] = v + return v + end + end) + -- comment removal + while true do + local np = op:gsub("%-%-[^\n]*\n", " ") + np = np:gsub("%-%-[^\n]*$", "") + if np == op then + break + end + op = np + end + -- stripping + while true do + local np = pass(op) + if np == op then + return np + end + op = np + end + return op + end end diff --git a/inst/status.lua b/inst/status.lua new file mode 100644 index 0000000..1acd41a --- /dev/null +++ b/inst/status.lua @@ -0,0 +1,36 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +-- Status Screen -- +local target = ... +local u = require("libs.frw") +local instSize = #u.read(target) + +local status = "" +local statusDetail = "" +local blinkI = "" +if instSize > 65536 then + blinkI = "5;31;" + status = " DO NOT SHIP " + statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code." +elseif instSize > 64000 then + blinkI = "33;" + status = " Shippable * " + statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes." +else + blinkI = "32;" + status = " All Green " + statusDetail = "The installer is well within budget with room for features.\nDevelop as normal." +end +io.stderr:write("\n") +local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n" +io.stderr:write(ctS .. " " .. ctM .. " " .. ctE) +io.stderr:write(ctS .. status .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE) +io.stderr:write(ctS .. " " .. ctM .. " " .. ctE) +io.stderr:write("\n") +io.stderr:write(statusDetail .. "\n") +io.stderr:write("\n") +io.stderr:write("Size: " .. instSize .. "\n") +io.stderr:write(" max. 65536\n") +io.stderr:write("\n") + diff --git a/inst/uncompressed/compress.lua b/inst/uncompressed/compress.lua index ff533ca..f0dd37d 100644 --- a/inst/uncompressed/compress.lua +++ b/inst/uncompressed/compress.lua @@ -1,7 +1,10 @@ -- This is released into the public domain. -- No warranty is provided, implied or otherwise. -return function (b) - return b +-- Example compression engine. +-- Given: data, lexCrunch +-- returns compressionEngine, compressedData +return function (data, lexCrunch) + return lexCrunch(" $engineInput = $engineOutput ", {}), data end diff --git a/inst/uncompressed/instdeco.lua b/inst/uncompressed/instdeco.lua deleted file mode 100644 index 6ed3281..0000000 --- a/inst/uncompressed/instdeco.lua +++ /dev/null @@ -1,13 +0,0 @@ --- This is released into the public domain. --- No warranty is provided, implied or otherwise. - -t = "" -function L(d) - if not d then return end - t = t .. d - while #t >= 512 do - M(t:sub(1, 512)) - t = t:sub(513) - end -end - diff --git a/inst/verify.lua b/inst/verify.lua deleted file mode 100644 index 30a4040..0000000 --- a/inst/verify.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This is released into the public domain. --- No warranty is provided, implied or otherwise. - --- Installer Compression Verification Tool -- -local alg, tarName = ... -local u = require("libs.frw") - -io.stderr:write("verifying... ") -local p = u.progress() - -local tarData = u.read(tarName) - -local total = "" -function M(t) - assert(#t == 512) - total = total .. t - p(#total / #tarData) -end - -dofile(alg .. "/instdeco.lua") - -L(u.read(alg .. "/output.bin")) - -if total ~= tarData then - io.stderr:write("\n" .. #total .. " : " .. #tarData .. "\n") - u.write(alg .. "/vfyerr.bin", total) - error("VERIFICATION FAILURE : see inst/" .. alg .. "/vfyerr.bin!") -end -io.stderr:write("\nverification success\n") - diff --git a/package.sh b/package.sh index fdaf2a7..fb9d406 100755 --- a/package.sh +++ b/package.sh @@ -16,7 +16,7 @@ cd .. # The Installer Creator cd inst lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua -lua verify.lua $1 ../code.tar +lua status.lua ../inst.lua cd .. # Common Repository Setup Code