Improve BDIVIDE (88 bytes free)

2020-03-30 17:47:27 +01:00 · 2020-03-30 17:47:27 +01:00 · 7840f0a231
parent 339571ee9b
commit 7840f0a231
9 changed files with 178 additions and 89 deletions
--- a/.gitignore
+++ b/.gitignore
@ -30,7 +30,10 @@ laboratory/*/*/*/*
 inst.lua
 # Available as the respective release
 inst-gold.lua
-inst/code.tar.bd
+# Compression stuff
+inst/*/output.bin
+inst/*/vfyerr.bin
+# internal
 upldr.sh
 upldr-dev.sh
 upldr-gold.sh
--- a/inst/bdivide/compress.lua
+++ b/inst/bdivide/compress.lua
@ -3,16 +3,12 @@

 -- PREPROC (r9 edition): preprocess input to be 7-bit

+local frw = require("libs.frw")
+
 local
 -- SHARED WITH DECOMPRESSION ENGINE
 function p(x, y)
 if x == 126 then
-  if y >= 32 then
-   return ({
-    -- Before adding to this, check how installer size is changed.
-    "\x7E", "\x7F"
-   })[y - 31], 3
-  end
  return string.char(y), 3
 elseif x == 127 then
  return string.char(128 + y), 3
@ -23,7 +19,7 @@ function p(x, y)
 elseif x == 30 then
  return "\x00", 2
 end
- return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte(math.floor(x / 5) + 1)), 2
+ return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2
 end

 local preprocParts = {}
@ -45,9 +41,10 @@ for i = 0, 127 do
 end
 end

-local function preproc(blk)
+local function preproc(blk, p)
 local out = ""
 while blk ~= "" do
+  p(blk)
  local len = math.min(preprocMaxLen, #blk)
  while len > 0 do
   local seg = blk:sub(1, len)
@ -71,7 +68,7 @@ end
 -- Position is where in the window it was found, minus 1.
 -- windowSize must be the same between the encoder and decoder,
 --  and is the amount of data preserved after cropping.
-local function bdivide(blk)
+local function bdivide(blk, p)
 local out = ""

 local windowSize = 0x10000
@ -82,6 +79,7 @@ local function bdivide(blk)
 end

 while blk ~= "" do
+  p(blk)
  local bestData = blk:sub(1, 1)
  local bestRes = bestData
  for lm = 0, 127 do
@ -111,8 +109,19 @@ local function bdivide(blk)
 end

 return function (data)
- data = preproc(data)
- io.stderr:write("---\n")
- data = bdivide(data)
- return data
+ io.stderr:write("preproc: ")
+ local pi = frw.progress()
+ local function p(b)
+  pi(1 - (#b / #data))
+ end
+ data = preproc(data, p)
+ io.stderr:write("\nbdivide: ")
+ pi = frw.progress()
+ data = bdivide(data, p)
+ io.stderr:write("\n")
+ -- These are used to pad the stream to flush the pipeline.
+ -- It's cheaper than the required code.
+ -- 1 byte of buffer for preproc,
+ -- 2 bytes of buffer for bdivide.
+ return data .. ("\x00"):rep(3)
 end
--- a/inst/bdivide/instdeco.lua
+++ b/inst/bdivide/instdeco.lua
@ -4,35 +4,35 @@
 -- BDIVIDE (r5 edition) and PREPROC (r9 edition)
 -- decompression engine for installer

-- cb: sector accumulator
-- ct: preproc accumulator
-- cc: bdivide accumulator
-- cw: bdivide window
+-- a: temporary
+--    touched by q,L
+-- b: sector accumulator
+-- c: bdivide accumulator
+-- d: temporary
+--    touched by q,L
+-- t: preproc accumulator
+-- q: function to submit to preproc
+-- s: temporary
+--    touched by L
+-- w: bdivide window

-- cp: function to submit to preproc
-- cd: function to submit to bdivide
+-- L: function to submit to bdivide

-cb,ct,cc,cw="","","",("\x00"):rep(2^16)
+b,t,c,w="","","",("\x00"):rep(2^16)
 -- High-level breakdown:
-- CP is unescaper & TAR-sector-breakup.
+-- q is unescaper & TAR-sector-breakup.
 -- It'll only begin to input if at least 3 bytes are available,
 --  so you'll want to throw in 2 extra zeroes at the end of stream as done here.
-- It uses Ct (input buffer) and Cp (output buffer).
+-- It uses t (input buffer) and p (output buffer).
 -- Ignore its second argument, as that's a lie, it's just there for a local.
-- CD is the actual decompressor. It has the same quirk as CP, wanting two more bytes.
-- It stores to Cc (compressed), and Cw (window).
-- It outputs that which goes to the window to CP also.
+-- L is the actual decompressor. It has the same quirk as q, wanting two more bytes.
+-- It stores to c (compressed), and w (window).
+-- It outputs that which goes to the window to q also.
 -- And it also uses a fake local.

 -- SEE compress.lua FOR THIS FUNCTION
 function p(x, y)
 if x == 126 then
-  if y >= 32 then
-   return ({
-    -- Before adding to this, check how installer size is changed.
-    "\x7E", "\x7F"
-   })[y - 31], 3
-  end
  return string.char(y), 3
 elseif x == 127 then
  return string.char(128 + y), 3
@ -43,43 +43,34 @@ function p(x, y)
 elseif x == 30 then
  return "\x00", 2
 end
- return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte(math.floor(x / 5) + 1)), 2
+ return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2
 end

-function cp(d, b, a)
- ct = ct .. d
- while #ct > 1 do
-  b, a = p(ct:byte(), ct:byte(2))
-  cb = cb .. b
-  ct = ct:sub(a)
-  if #cb > 511 then
-   M(cb:sub(1, 512))
-   cb = cb:sub(513)
+function q(w)
+ t = t .. w
+ while #t > 1 do
+  d, a = p(t:byte(), t:byte(2))
+  b = b .. d
+  t = t:sub(a)
+  if #b > 511 then
+   M(b:sub(1, 512))
+   b = b:sub(513)
  end
 end
 end

-function cd(d, b, p)
- cc = cc .. d
- while #cc > 2 do
-  b = cc:byte()
-  if b < 128 then
-   b, cc = cc:sub(1, 1), cc:sub(2)
-  else
-   p = cc:byte(2) * 256 + cc:byte(3) + 1
-   b, cc = cw:sub(p, p + b - 125), cc:sub(4)
-  end
-  cp(b)
-  cw = (cw .. b):sub(-65536)
- end
-end
-
-- quick & dirty integration with the existing stuff
 function L(d)
- if not d then
-  cd("\x00\x00")cp("\x00\x00")
- else
-  cd(d)
+ c = c .. d
+ while #c > 2 do
+  s = c:byte()
+  if s < 128 then
+   s, c = c:sub(1, 1), c:sub(2)
+  else
+   a = c:byte(2) * 256 + c:byte(3) + 1
+   s, c = w:sub(a, a + s - 125), c:sub(4)
+  end
+  q(s)
+  w = (w .. s):sub(-2^16)
 end
 end

--- a/inst/build.lua
+++ b/inst/build.lua
@ -2,30 +2,66 @@
 -- No warranty is provided, implied or otherwise.

 -- KittenOS NEO Installer Generator --
-local alg, tarName = ...
+local alg, tarName, cid = ...
+cid = (cid or "UNKNOWN"):sub(1, 7)

-local function read(fn)
- local f = io.open(fn, "rb")
- local d = f:read("*a")
- f:close()
- return d
-end
+local u = require("libs.frw")

-local tarData = read(tarName)
+local tarData = u.read(tarName)
 local tarSectors = math.floor(#tarData / 512)

-local instCode = "K=" .. tarSectors .. "\n" .. read(alg .. "/instdeco.lua") .. read("instbase.lua")
+local instSize = 0
+local function put(data)
+ io.write(data)
+ instSize = instSize + #data
+end
+
+put("--" .. cid .. "\n")
+put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
+
+local instCode = "K=" .. tarSectors .. "\n" .. u.read(alg .. "/instdeco.lua") .. u.read("instbase.lua")
 instCode = require("libs.lexcrunch")(instCode)
-io.write(instCode)
+put(instCode)

 -- the \x00 is the indicator to start reading
-io.write("--[[\x00")
+put("--[[\x00")
+
 io.stderr:write("compressing...\n")
 local compressedData = require(alg .. ".compress")(tarData)
+u.write(alg .. "/output.bin", compressedData)
 io.stderr:write("compression with " .. alg .. ": " .. #tarData .. " -> " .. #compressedData .. "\n")
 -- Program the read-in state machine
 compressedData = compressedData:gsub("\xFE", "\xFE\xFE")
 compressedData = compressedData:gsub("]]", "]\xFE]")
-io.write(compressedData)
-io.write("]]")
+
+put(compressedData)
+put("]]")
+
+local status = ""
+local statusDetail = ""
+local blinkI = ""
+if instSize > 65536 then
+ blinkI = "5;31;"
+ status = " DO NOT SHIP "
+ statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
+elseif instSize > 64000 then
+ blinkI = "33;"
+ status = " Shippable * "
+ statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
+else
+ blinkI = "32;"
+ status = "  All Green  "
+ statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
+end
+io.stderr:write("\n")
+local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
+io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
+io.stderr:write(ctS .. status          .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
+io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
+io.stderr:write("\n")
+io.stderr:write(statusDetail .. "\n")
+io.stderr:write("\n")
+io.stderr:write("Size:  " .. instSize .. "\n")
+io.stderr:write(" max.  65536\n")
+io.stderr:write("\n")

--- a/inst/instbase.lua
+++ b/inst/instbase.lua
@ -33,7 +33,7 @@
 -- Z: component: filesystem
 B = computer
 C = component
-assert(C, "To install, please copy as init.lua to a blank disk or a system to update, then remove all other disks and reboot.")
+assert(C, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")

 X = C.list("screen", true)()
 Y = C.list("gpu", true)()
@ -113,12 +113,6 @@ end

 while true do
 A = Z.read(P, 64)
- if not A then
-  L()
-  -- IF WE GET HERE,
-  -- YOU BROKE SOMETHING!
-  A()
- end
 D = ""
 for i = 1, #A do
  -- Read-in state machine
--- a/inst/libs/frw.lua
+++ b/inst/libs/frw.lua
@ -0,0 +1,30 @@
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+return {
+ read = function (fn)
+  local f = io.open(fn, "rb")
+  local d = f:read("*a")
+  f:close()
+  return d
+ end,
+ write = function (fn, data)
+  local f = io.open(fn, "wb")
+  f:write(data)
+  f:close()
+ end,
+ progress = function ()
+  io.stderr:write("00% \\")
+  local lastPercent = 0
+  local chr = 0
+  return function (fraction)
+   local percent = math.ceil(fraction * 100)
+   if percent ~= lastPercent then
+    lastPercent = percent
+    chr = (chr + 1) % 4
+    io.stderr:write(string.format("\8\8\8\8\8%02i%% %s", percent, ("\\|/-"):sub(chr + 1, chr + 1)))
+   end
+  end
+ end
+}
+
--- a/inst/uncompressed/instdeco.lua
+++ b/inst/uncompressed/instdeco.lua
@ -3,6 +3,7 @@

 t = ""
 function L(d)
+ if not d then return end
 t = t .. d
 while #t >= 512 do
  M(t:sub(1, 512))
--- a/inst/verify.lua
+++ b/inst/verify.lua
@ -0,0 +1,30 @@
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- Installer Compression Verification Tool --
+local alg, tarName = ...
+local u = require("libs.frw")
+
+io.stderr:write("verifying... ")
+local p = u.progress()
+
+local tarData = u.read(tarName)
+
+local total = ""
+function M(t)
+ assert(#t == 512)
+ total = total .. t
+ p(#total / #tarData)
+end
+
+dofile(alg .. "/instdeco.lua")
+
+L(u.read(alg .. "/output.bin"))
+
+if total ~= tarData then
+ io.stderr:write("\n" .. #total .. " : " .. #tarData .. "\n")
+ u.write(alg .. "/vfyerr.bin", total)
+ error("VERIFICATION FAILURE : see inst/" .. alg .. "/vfyerr.bin!")
+end
+io.stderr:write("\nverification success\n")
+
--- a/package.sh
+++ b/package.sh
@ -13,15 +13,10 @@ cd code
 tar --mtime=0 --owner=gray:0 --group=mann:0 -cf ../code.tar .
 cd ..

-# Solely for ensuring that a -gold.lua file can be checked before being pushed to repository.
-echo -n "-- KOSNEO inst. " > inst.lua
-git status --porcelain=2 --branch | grep branch.oid >> inst.lua
-echo "-- This is released into the public domain." >> inst.lua
-echo "-- No warranty is provided, implied or otherwise." >> inst.lua
-
 # The Installer Creator
 cd inst
-lua build.lua $1 ../code.tar >> ../inst.lua
+lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua
+lua verify.lua $1 ../code.tar
 cd ..

 # Common Repository Setup Code