From 7840f0a231e0cd50df25059ba81e7dcf832f5691 Mon Sep 17 00:00:00 2001 From: 20kdc Date: Mon, 30 Mar 2020 17:47:27 +0100 Subject: [PATCH] Improve BDIVIDE (88 bytes free) --- .gitignore | 5 +- inst/bdivide/compress.lua | 35 +++++++++----- inst/bdivide/instdeco.lua | 87 +++++++++++++++------------------- inst/build.lua | 62 +++++++++++++++++++----- inst/instbase.lua | 8 +--- inst/libs/frw.lua | 30 ++++++++++++ inst/uncompressed/instdeco.lua | 1 + inst/verify.lua | 30 ++++++++++++ package.sh | 9 +--- 9 files changed, 178 insertions(+), 89 deletions(-) create mode 100644 inst/libs/frw.lua create mode 100644 inst/verify.lua diff --git a/.gitignore b/.gitignore index 9de53ff..eef518f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,10 @@ laboratory/*/*/*/* inst.lua # Available as the respective release inst-gold.lua -inst/code.tar.bd +# Compression stuff +inst/*/output.bin +inst/*/vfyerr.bin +# internal upldr.sh upldr-dev.sh upldr-gold.sh diff --git a/inst/bdivide/compress.lua b/inst/bdivide/compress.lua index b0a9f99..0b76f8f 100644 --- a/inst/bdivide/compress.lua +++ b/inst/bdivide/compress.lua @@ -3,16 +3,12 @@ -- PREPROC (r9 edition): preprocess input to be 7-bit +local frw = require("libs.frw") + local -- SHARED WITH DECOMPRESSION ENGINE function p(x, y) if x == 126 then - if y >= 32 then - return ({ - -- Before adding to this, check how installer size is changed. - "\x7E", "\x7F" - })[y - 31], 3 - end return string.char(y), 3 elseif x == 127 then return string.char(128 + y), 3 @@ -23,7 +19,7 @@ function p(x, y) elseif x == 30 then return "\x00", 2 end - return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte(math.floor(x / 5) + 1)), 2 + return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2 end local preprocParts = {} @@ -45,9 +41,10 @@ for i = 0, 127 do end end -local function preproc(blk) +local function preproc(blk, p) local out = "" while blk ~= "" do + p(blk) local len = math.min(preprocMaxLen, #blk) while len > 0 do local seg = blk:sub(1, len) @@ -71,7 +68,7 @@ end -- Position is where in the window it was found, minus 1. -- windowSize must be the same between the encoder and decoder, -- and is the amount of data preserved after cropping. -local function bdivide(blk) +local function bdivide(blk, p) local out = "" local windowSize = 0x10000 @@ -82,6 +79,7 @@ local function bdivide(blk) end while blk ~= "" do + p(blk) local bestData = blk:sub(1, 1) local bestRes = bestData for lm = 0, 127 do @@ -111,8 +109,19 @@ local function bdivide(blk) end return function (data) - data = preproc(data) - io.stderr:write("---\n") - data = bdivide(data) - return data + io.stderr:write("preproc: ") + local pi = frw.progress() + local function p(b) + pi(1 - (#b / #data)) + end + data = preproc(data, p) + io.stderr:write("\nbdivide: ") + pi = frw.progress() + data = bdivide(data, p) + io.stderr:write("\n") + -- These are used to pad the stream to flush the pipeline. + -- It's cheaper than the required code. + -- 1 byte of buffer for preproc, + -- 2 bytes of buffer for bdivide. + return data .. ("\x00"):rep(3) end diff --git a/inst/bdivide/instdeco.lua b/inst/bdivide/instdeco.lua index 6ecdd6c..60cf99a 100644 --- a/inst/bdivide/instdeco.lua +++ b/inst/bdivide/instdeco.lua @@ -4,35 +4,35 @@ -- BDIVIDE (r5 edition) and PREPROC (r9 edition) -- decompression engine for installer --- cb: sector accumulator --- ct: preproc accumulator --- cc: bdivide accumulator --- cw: bdivide window +-- a: temporary +-- touched by q,L +-- b: sector accumulator +-- c: bdivide accumulator +-- d: temporary +-- touched by q,L +-- t: preproc accumulator +-- q: function to submit to preproc +-- s: temporary +-- touched by L +-- w: bdivide window --- cp: function to submit to preproc --- cd: function to submit to bdivide +-- L: function to submit to bdivide -cb,ct,cc,cw="","","",("\x00"):rep(2^16) +b,t,c,w="","","",("\x00"):rep(2^16) -- High-level breakdown: --- CP is unescaper & TAR-sector-breakup. +-- q is unescaper & TAR-sector-breakup. -- It'll only begin to input if at least 3 bytes are available, -- so you'll want to throw in 2 extra zeroes at the end of stream as done here. --- It uses Ct (input buffer) and Cp (output buffer). +-- It uses t (input buffer) and p (output buffer). -- Ignore its second argument, as that's a lie, it's just there for a local. --- CD is the actual decompressor. It has the same quirk as CP, wanting two more bytes. --- It stores to Cc (compressed), and Cw (window). --- It outputs that which goes to the window to CP also. +-- L is the actual decompressor. It has the same quirk as q, wanting two more bytes. +-- It stores to c (compressed), and w (window). +-- It outputs that which goes to the window to q also. -- And it also uses a fake local. -- SEE compress.lua FOR THIS FUNCTION function p(x, y) if x == 126 then - if y >= 32 then - return ({ - -- Before adding to this, check how installer size is changed. - "\x7E", "\x7F" - })[y - 31], 3 - end return string.char(y), 3 elseif x == 127 then return string.char(128 + y), 3 @@ -43,43 +43,34 @@ function p(x, y) elseif x == 30 then return "\x00", 2 end - return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte(math.floor(x / 5) + 1)), 2 + return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2 end -function cp(d, b, a) - ct = ct .. d - while #ct > 1 do - b, a = p(ct:byte(), ct:byte(2)) - cb = cb .. b - ct = ct:sub(a) - if #cb > 511 then - M(cb:sub(1, 512)) - cb = cb:sub(513) +function q(w) + t = t .. w + while #t > 1 do + d, a = p(t:byte(), t:byte(2)) + b = b .. d + t = t:sub(a) + if #b > 511 then + M(b:sub(1, 512)) + b = b:sub(513) end end end -function cd(d, b, p) - cc = cc .. d - while #cc > 2 do - b = cc:byte() - if b < 128 then - b, cc = cc:sub(1, 1), cc:sub(2) - else - p = cc:byte(2) * 256 + cc:byte(3) + 1 - b, cc = cw:sub(p, p + b - 125), cc:sub(4) - end - cp(b) - cw = (cw .. b):sub(-65536) - end -end - --- quick & dirty integration with the existing stuff function L(d) - if not d then - cd("\x00\x00")cp("\x00\x00") - else - cd(d) + c = c .. d + while #c > 2 do + s = c:byte() + if s < 128 then + s, c = c:sub(1, 1), c:sub(2) + else + a = c:byte(2) * 256 + c:byte(3) + 1 + s, c = w:sub(a, a + s - 125), c:sub(4) + end + q(s) + w = (w .. s):sub(-2^16) end end diff --git a/inst/build.lua b/inst/build.lua index 7454a68..de464f7 100644 --- a/inst/build.lua +++ b/inst/build.lua @@ -2,30 +2,66 @@ -- No warranty is provided, implied or otherwise. -- KittenOS NEO Installer Generator -- -local alg, tarName = ... +local alg, tarName, cid = ... +cid = (cid or "UNKNOWN"):sub(1, 7) -local function read(fn) - local f = io.open(fn, "rb") - local d = f:read("*a") - f:close() - return d -end +local u = require("libs.frw") -local tarData = read(tarName) +local tarData = u.read(tarName) local tarSectors = math.floor(#tarData / 512) -local instCode = "K=" .. tarSectors .. "\n" .. read(alg .. "/instdeco.lua") .. read("instbase.lua") +local instSize = 0 +local function put(data) + io.write(data) + instSize = instSize + #data +end + +put("--" .. cid .. "\n") +put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n") + +local instCode = "K=" .. tarSectors .. "\n" .. u.read(alg .. "/instdeco.lua") .. u.read("instbase.lua") instCode = require("libs.lexcrunch")(instCode) -io.write(instCode) +put(instCode) -- the \x00 is the indicator to start reading -io.write("--[[\x00") +put("--[[\x00") + io.stderr:write("compressing...\n") local compressedData = require(alg .. ".compress")(tarData) +u.write(alg .. "/output.bin", compressedData) io.stderr:write("compression with " .. alg .. ": " .. #tarData .. " -> " .. #compressedData .. "\n") -- Program the read-in state machine compressedData = compressedData:gsub("\xFE", "\xFE\xFE") compressedData = compressedData:gsub("]]", "]\xFE]") -io.write(compressedData) -io.write("]]") + +put(compressedData) +put("]]") + +local status = "" +local statusDetail = "" +local blinkI = "" +if instSize > 65536 then + blinkI = "5;31;" + status = " DO NOT SHIP " + statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code." +elseif instSize > 64000 then + blinkI = "33;" + status = " Shippable * " + statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes." +else + blinkI = "32;" + status = " All Green " + statusDetail = "The installer is well within budget with room for features.\nDevelop as normal." +end +io.stderr:write("\n") +local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n" +io.stderr:write(ctS .. " " .. ctM .. " " .. ctE) +io.stderr:write(ctS .. status .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE) +io.stderr:write(ctS .. " " .. ctM .. " " .. ctE) +io.stderr:write("\n") +io.stderr:write(statusDetail .. "\n") +io.stderr:write("\n") +io.stderr:write("Size: " .. instSize .. "\n") +io.stderr:write(" max. 65536\n") +io.stderr:write("\n") diff --git a/inst/instbase.lua b/inst/instbase.lua index 2757e7b..1c61180 100644 --- a/inst/instbase.lua +++ b/inst/instbase.lua @@ -33,7 +33,7 @@ -- Z: component: filesystem B = computer C = component -assert(C, "To install, please copy as init.lua to a blank disk or a system to update, then remove all other disks and reboot.") +assert(C, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.") X = C.list("screen", true)() Y = C.list("gpu", true)() @@ -113,12 +113,6 @@ end while true do A = Z.read(P, 64) - if not A then - L() - -- IF WE GET HERE, - -- YOU BROKE SOMETHING! - A() - end D = "" for i = 1, #A do -- Read-in state machine diff --git a/inst/libs/frw.lua b/inst/libs/frw.lua new file mode 100644 index 0000000..272c914 --- /dev/null +++ b/inst/libs/frw.lua @@ -0,0 +1,30 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +return { + read = function (fn) + local f = io.open(fn, "rb") + local d = f:read("*a") + f:close() + return d + end, + write = function (fn, data) + local f = io.open(fn, "wb") + f:write(data) + f:close() + end, + progress = function () + io.stderr:write("00% \\") + local lastPercent = 0 + local chr = 0 + return function (fraction) + local percent = math.ceil(fraction * 100) + if percent ~= lastPercent then + lastPercent = percent + chr = (chr + 1) % 4 + io.stderr:write(string.format("\8\8\8\8\8%02i%% %s", percent, ("\\|/-"):sub(chr + 1, chr + 1))) + end + end + end +} + diff --git a/inst/uncompressed/instdeco.lua b/inst/uncompressed/instdeco.lua index ea9a07b..6ed3281 100644 --- a/inst/uncompressed/instdeco.lua +++ b/inst/uncompressed/instdeco.lua @@ -3,6 +3,7 @@ t = "" function L(d) + if not d then return end t = t .. d while #t >= 512 do M(t:sub(1, 512)) diff --git a/inst/verify.lua b/inst/verify.lua new file mode 100644 index 0000000..30a4040 --- /dev/null +++ b/inst/verify.lua @@ -0,0 +1,30 @@ +-- This is released into the public domain. +-- No warranty is provided, implied or otherwise. + +-- Installer Compression Verification Tool -- +local alg, tarName = ... +local u = require("libs.frw") + +io.stderr:write("verifying... ") +local p = u.progress() + +local tarData = u.read(tarName) + +local total = "" +function M(t) + assert(#t == 512) + total = total .. t + p(#total / #tarData) +end + +dofile(alg .. "/instdeco.lua") + +L(u.read(alg .. "/output.bin")) + +if total ~= tarData then + io.stderr:write("\n" .. #total .. " : " .. #tarData .. "\n") + u.write(alg .. "/vfyerr.bin", total) + error("VERIFICATION FAILURE : see inst/" .. alg .. "/vfyerr.bin!") +end +io.stderr:write("\nverification success\n") + diff --git a/package.sh b/package.sh index e2017a7..fdaf2a7 100755 --- a/package.sh +++ b/package.sh @@ -13,15 +13,10 @@ cd code tar --mtime=0 --owner=gray:0 --group=mann:0 -cf ../code.tar . cd .. -# Solely for ensuring that a -gold.lua file can be checked before being pushed to repository. -echo -n "-- KOSNEO inst. " > inst.lua -git status --porcelain=2 --branch | grep branch.oid >> inst.lua -echo "-- This is released into the public domain." >> inst.lua -echo "-- No warranty is provided, implied or otherwise." >> inst.lua - # The Installer Creator cd inst -lua build.lua $1 ../code.tar >> ../inst.lua +lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua +lua verify.lua $1 ../code.tar cd .. # Common Repository Setup Code