The DEFLATE Decompressor (Of Pain And Suffering)

code/apps/sys-everest.lua

@@ -611,57 +611,44 @@ local function key(ku, ka, kc, down)
  lIM = lin
 
  local focus = surfaces[1]
- if kc == 29 then isCtrDown = down end
- if kc == 56 then isAltDown = down end
- if isAltDown then
-  if ka == 120 then
-   if focus and down then ofsMSurface(focus, 1) end return
-  end
-  if kc == 200 then
-   if focus and down then ofsSurface(focus, 0, -1) end return
-  end
-  if kc == 208 then
-   if focus and down then ofsSurface(focus, 0, 1) end return
-  end
-  if kc == 203 then
-   if focus and down then ofsSurface(focus, -1, 0) end return
-  end
-  if kc == 205 then
-   if focus and down then ofsSurface(focus, 1, 0) end return
-  end
-  if ka == 122 then
-   if focus and down then
-    local n = table.remove(surfaces, 1)
-    table.insert(surfaces, n)
-    changeFocus(n)
-   end return
-  end
-  if ka == 97 then
-   if not down then
-    isAltDown = false
-   end
-   return
-  end
-  if ka == 3 or ka == 99 then
-   if down then
-    if isCtrDown then
-     error("User-authorized Everest crash.")
-    else
-     if focus then
-      focus[6](focus[8], "close")
-     end
-    end
-   end
-   return
-  end
-  if ka == 13 then
-   if down then
-    startLauncher()
-   end
-   return
-  end
+ if kc == 29 then
+  isCtrDown = down
+ elseif kc == 56 then
+  isAltDown = down
  end
- if focus then
+ if isAltDown and kc == 122 then
+  if focus and down then
+   local n = table.remove(surfaces, 1)
+   table.insert(surfaces, n)
+   changeFocus(n)
+  end
+ elseif isAltDown and kc == 200 then
+  if focus and down then ofsSurface(focus, 0, -1) end
+ elseif isAltDown and kc == 208 then
+  if focus and down then ofsSurface(focus, 0, 1) end
+ elseif isAltDown and kc == 203 then
+  if focus and down then ofsSurface(focus, -1, 0) end
+ elseif isAltDown and kc == 205 then
+  if focus and down then ofsSurface(focus, 1, 0) end
+ elseif isAltDown and ka == 120 then
+  if focus and down then ofsMSurface(focus, 1) end
+ elseif isAltDown and ka == 97 then
+  if not down then
+   isAltDown = false
+  end
+ elseif isAltDown and (ka == 3 or ka == 99) then
+  if down then
+   if isCtrDown then
+    error("User-authorized Everest crash.")
+   elseif focus then
+    focus[6](focus[8], "close")
+   end
+  end
+ elseif isAltDown and ka == 13 then
+  if down then
+   startLauncher()
+  end
+ elseif focus then
   if kc ~= 56 then
    lIM = focus[1]
   end

inst/bdivide/compress.lua

@@ -74,10 +74,6 @@ local function bdivide(blk, p)
  local windowSize = 0x10000
  local windowData = ("\x00"):rep(windowSize)
 
- local function crop(data)
-  windowData = (windowData .. data):sub(-windowSize)
- end
-
  while blk ~= "" do
   p(blk)
   local bestData = blk:sub(1, 1)
@@ -102,13 +98,14 @@ local function bdivide(blk, p)
   end
   -- ok, encode!
   out = out .. bestData
-  crop(bestRes)
+  -- crop window
+  windowData = (windowData .. bestRes):sub(-windowSize)
   blk = blk:sub(#bestRes + 1)
  end
  return out
 end
 
-return function (data)
+return function (data, lexCrunch)
  io.stderr:write("preproc: ")
  local pi = frw.progress()
  local function p(b)
@@ -123,5 +120,5 @@ return function (data)
  -- It's cheaper than the required code.
  -- 1 byte of buffer for preproc,
  -- 2 bytes of buffer for bdivide.
- return data .. ("\x00"):rep(3)
+ return lexCrunch(frw.read("bdivide/instdeco.lua"), {}), data .. ("\x00"):rep(3)
 end

inst/bdivide/instdeco.lua

@@ -4,23 +4,11 @@
 -- BDIVIDE (r5 edition) and PREPROC (r9 edition)
 -- decompression engine for installer
 
--- a: temporary
---    touched by q,L
--- b: sector accumulator
--- c: bdivide accumulator
--- d: temporary
---    touched by q,L
--- t: preproc accumulator
--- q: function to submit to preproc
--- s: temporary
---    touched by L
--- w: bdivide window
-
--- L: function to submit to bdivide
-
-b,t,c,w="","","",("\x00"):rep(2^16)
+$bdPPBuffer = ""
+$bdBDBuffer = ""
+$bdBDWindow = ("\x00"):rep(2^16)
 -- High-level breakdown:
--- q is unescaper & TAR-sector-breakup.
+-- q is unescaper.
 -- It'll only begin to input if at least 3 bytes are available,
 --  so you'll want to throw in 2 extra zeroes at the end of stream as done here.
 -- It uses t (input buffer) and p (output buffer).
@@ -31,7 +19,7 @@ b,t,c,w="","","",("\x00"):rep(2^16)
 -- And it also uses a fake local.
 
 -- SEE compress.lua FOR THIS FUNCTION
-function p(x, y)
+function $bdPP(x, y)
  if x == 126 then
   return string.char(y), 3
  elseif x == 127 then
@@ -46,31 +34,29 @@ function p(x, y)
  return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2
 end
 
-function q(w)
- t = t .. w
- while #t > 1 do
-  d, a = p(t:byte(), t:byte(2))
-  b = b .. d
-  t = t:sub(a)
-  if #b > 511 then
-   M(b:sub(1, 512))
-   b = b:sub(513)
-  end
- end
-end
-
-function L(d)
- c = c .. d
- while #c > 2 do
-  s = c:byte()
-  if s < 128 then
-   s, c = c:sub(1, 1), c:sub(2)
+function $engineInput($a0)
+ $bdBDBuffer = $bdBDBuffer .. $a0
+ while #$bdBDBuffer > 2 do
+  $a0 = $bdBDBuffer:byte()
+  if $a0 < 128 then
+   $a0 = $bdBDBuffer:sub(1, 1)
+   $bdBDBuffer = $bdBDBuffer:sub(2)
   else
-   a = c:byte(2) * 256 + c:byte(3) + 1
-   s, c = w:sub(a, a + s - 125), c:sub(4)
+   $NTbdBDPtr
+   $bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1
+   $a0 = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $a0 - 125)
+   $bdBDBuffer = $bdBDBuffer:sub(4)
+   $DTbdBDPtr
+  end
+  $bdPPBuffer = $bdPPBuffer .. $a0
+  $bdBDWindow = ($bdBDWindow .. $a0):sub(-2^16)
+  while #$bdPPBuffer > 1 do
+   $NTbdPPAdv
+   $a0, $bdPPAdv = $bdPP($bdPPBuffer:byte(), $bdPPBuffer:byte(2))
+   $bdPPBuffer = $bdPPBuffer:sub($bdPPAdv)
+   $DTbdPPAdv
+   $engineOutput($a0)
   end
-  q(s)
-  w = (w .. s):sub(-2^16)
  end
 end
 

inst/build.lua

@@ -7,8 +7,7 @@ cid = (cid or "UNKNOWN"):sub(1, 7)
 
 local u = require("libs.frw")
 
-local tarData = u.read(tarName)
-local tarSectors = math.floor(#tarData / 512)
+local algImpl = require(alg .. ".compress")
 
 local instSize = 0
 local function put(data)
@@ -16,52 +15,31 @@ local function put(data)
  instSize = instSize + #data
 end
 
-put("--" .. cid .. "\n")
-put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
+-- TAR File --
+local tarData = u.read(tarName)
+local tarSectors = math.floor(#tarData / 512)
 
-local instCode = "K=" .. tarSectors .. "\n" .. u.read(alg .. "/instdeco.lua") .. u.read("instbase.lua")
-instCode = require("libs.lexcrunch")(instCode)
-put(instCode)
+-- Installer Lexcrunch Context --
+local lexCrunch = require("libs.lexcrunch")()
 
--- the \x00 is the indicator to start reading
-put("--[[\x00")
+local installerCore = lexCrunch(u.read("instcore.lua"), {["$$SECTORS"] = tostring(tarSectors)})
+local installerHead = lexCrunch(u.read("insthead.lua"), {["$$CORESIZE"] = tostring(#installerCore)})
+local installerTail = lexCrunch(u.read("insttail.lua"), {})
 
+-- Installer Compression --
+local rawData = installerCore .. tarData
 io.stderr:write("compressing...\n")
-local compressedData = require(alg .. ".compress")(tarData)
-u.write(alg .. "/output.bin", compressedData)
-io.stderr:write("compression with " .. alg .. ": " .. #tarData .. " -> " .. #compressedData .. "\n")
--- Program the read-in state machine
+local compressionEngine, compressedData = algImpl(rawData, lexCrunch)
+-- RISM [[
 compressedData = compressedData:gsub("\xFE", "\xFE\xFE")
 compressedData = compressedData:gsub("]]", "]\xFE]")
+compressedData = "\x00" .. compressedData
+-- ]]
+io.stderr:write("compression with " .. alg .. ": " .. #rawData .. " -> " .. #compressedData .. "\n")
 
-put(compressedData)
-put("]]")
-
-local status = ""
-local statusDetail = ""
-local blinkI = ""
-if instSize > 65536 then
- blinkI = "5;31;"
- status = " DO NOT SHIP "
- statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
-elseif instSize > 64000 then
- blinkI = "33;"
- status = " Shippable * "
- statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
-else
- blinkI = "32;"
- status = "  All Green  "
- statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
-end
-io.stderr:write("\n")
-local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
-io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
-io.stderr:write(ctS .. status          .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
-io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
-io.stderr:write("\n")
-io.stderr:write(statusDetail .. "\n")
-io.stderr:write("\n")
-io.stderr:write("Size:  " .. instSize .. "\n")
-io.stderr:write(" max.  65536\n")
-io.stderr:write("\n")
+-- Installer Final Generation --
+put("--" .. cid .. "\n")
+put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
+put(lexCrunch(installerHead .. compressionEngine .. installerTail, {}))
+put("--[[" .. compressedData .. "]]")
 

inst/deflate/compress.lua

@@ -0,0 +1,14 @@
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- This is a wrapper around (i.e. does not contain) Zopfli.
+local frw = require("libs.frw")
+
+return function (data, lexCrunch)
+ frw.write("tempData.bin", data)
+ os.execute("zopfli --i1 --deflate -c tempData.bin > tempZopf.bin")
+ local res = frw.read("tempZopf.bin")
+ os.execute("rm tempData.bin tempZopf.bin")
+ return lexCrunch(frw.read("deflate/instdeco.lua"), {}), res
+end
+

inst/deflate/instdeco.lua

@@ -0,0 +1,263 @@
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- THE DEFLATE DECOMPRESSOR OF MADNESS --
+
+-- Core I/O --
+
+-- $dfAlignToByteRemaining is
+--  set to 8 in the outer engine
+
+function $dfGetIntField($a0, $a1)
+ $NTdfForLoopVar
+ $a1 = 0
+ for $dfForLoopVar = 1, $a0 do
+  if coroutine.yield() then
+   $a1 = $a1 + 2^($dfForLoopVar - 1)
+  end
+ end
+ $DTdfForLoopVar
+ return $a1
+end
+
+function $dfAlignToByte()
+ while $dfAlignToByteRemaining do
+  coroutine.yield()
+ end
+end
+
+-- Huffman Core --
+-- The approach here is based around 1-prefixed integers.
+-- These are stored in a flat table.
+-- So 0b1000 is the pattern 000.
+
+function $dfReadHuffmanSymbol($a0, $a1)
+ $a1 = 1
+ while not $a0[$a1] do
+  $a1 = ($a1 * 2) + $dfGetIntField(1)
+ end
+ return $a0[$a1]
+end
+
+function $dfGenHuffmanTree($a0)
+ local blCount = {}
+ -- Note the 0
+ for loopVar = 0, 15 do
+  blCount[loopVar] = 0
+ end
+ for loopVar = 0, #$a0 do
+  local cl = $a0[loopVar]
+  if cl ~= 0 then
+   blCount[cl] = blCount[cl] + 1
+  end
+ end
+
+ local code = 0
+ local nextCode = {}
+ for loopVar = 1, 15 do
+  code = (code + blCount[loopVar - 1]) * 2
+  nextCode[loopVar] = code
+ end
+
+ local tbl = {}
+ for loopVar = 0, #$a0 do
+  local cl = $a0[loopVar]
+  if cl ~= 0 then
+   local pow = math.floor(2 ^ cl)
+   if nextCode[cl] >= pow then error("To L1 not valid for " .. n .. "," .. cl .. ".") end
+   local k = nextCode[cl] + pow
+   assert(not tbl[k], "conflict @ " .. k)
+   tbl[k] = loopVar
+   nextCode[cl] = nextCode[cl] + 1
+  end
+ end
+ return tbl
+end
+
+-- DEFLATE fixed trees --
+$NTdfFixedTL
+$NTdfFlv
+$dfFixedTL = {}
+for $dfFlv = 0, 143 do $dfFixedTL[$dfFlv] = 8 end
+for $dfFlv = 144, 255 do $dfFixedTL[$dfFlv] = 9 end
+for $dfFlv = 256, 279 do $dfFixedTL[$dfFlv] = 7 end
+for $dfFlv = 280, 287 do $dfFixedTL[$dfFlv] = 8 end
+$dfFixedLit = $dfGenHuffmanTree($dfFixedTL)
+$dfFixedTL = {}
+for $dfFlv = 0, 31 do
+ $dfFixedTL[$dfFlv] = 5
+end
+$dfFixedDst = $dfGenHuffmanTree($dfFixedTL)
+$DTdfFlv
+$DTdfFixedTL
+
+-- DEFLATE LZ Core --
+
+$dfWindow = ("\x00"):rep(2^16)
+$dfPushBuf = ""
+function $dfOutput($a0)
+ $dfWindow = ($dfWindow .. $a0):sub(-2^16)
+ $dfPushBuf = $dfPushBuf .. $a0
+end
+
+$dfBittblLength = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 2, 2, 2, 2,
+ 3, 3, 3, 3, 4, 4, 4, 4,
+ 5, 5, 5, 5, 0
+}
+$dfBasetblLength = {
+ 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 13, 15, 17, 19, 23, 27, 31,
+ 35, 43, 51, 59, 67, 83, 99, 115,
+ 131, 163, 195, 227, 258
+}
+$dfBittblDist = {
+ 0, 0, 0, 0, 1, 1, 2, 2,
+ 3, 3, 4, 4, 5, 5, 6, 6,
+ 7, 7, 8, 8, 9, 9, 10, 10,
+ 11, 11, 12, 12, 13, 13
+}
+$dfBasetblDist = {
+ 1, 2, 3, 4, 5, 7, 9, 13,
+ 17, 25, 33, 49, 65, 97, 129, 193,
+ 257, 385, 513, 769, 1025, 1537, 2049, 3073,
+ 4097, 6145, 8193, 12289, 16385, 24577
+}
+
+function $dfReadBlockBody(lit, dst)
+ while true do
+  local litSym = $dfReadHuffmanSymbol(lit)
+  if litSym <= 255 then
+   $dfOutput(string.char(litSym))
+  elseif litSym == 256 then
+   return
+  elseif litSym <= 285 then
+   local len = $dfBasetblLength[litSym - 256] + $dfGetIntField($dfBittblLength[litSym - 256])
+   local dCode = $dfReadHuffmanSymbol(dst)
+   local dst = $dfBasetblDist[dCode + 1] + $dfGetIntField($dfBittblDist[dCode + 1])
+   local ptr = 65537 - dst
+   for loopVar = 1, len do
+    $dfOutput($dfWindow:sub(ptr, ptr))
+   end
+  else
+   error("nt" .. v)
+  end
+ end
+end
+
+-- Huffman Dynamics --
+
+function $dfReadHuffmanDynamicSubcodes(distlens, dst, metatree)
+ local loopVar = 0
+ distlens[-1] = 0
+ while loopVar < dst do
+  local instr = $dfReadHuffmanSymbol(metatree)
+  if instr < 16 then
+   distlens[loopVar] = instr
+   loopVar = loopVar + 1
+  elseif instr == 16 then
+   for loopVar2 = 1, 3 + $dfGetIntField(2) do
+    distlens[loopVar] = distlens[loopVar - 1]
+    loopVar = loopVar + 1
+    if loopVar > dst then error("Overflow") end
+   end
+  elseif instr == 17 then
+   for loopVar2 = 1, 3 + $dfGetIntField(3) do
+    distlens[loopVar] = 0
+    loopVar = loopVar + 1
+    if loopVar > dst then error("Overflow") end
+   end
+  elseif instr == 18 then
+   for loopVar2 = 1, 11 + $dfGetIntField(7) do
+    distlens[loopVar] = 0
+    loopVar = loopVar + 1
+    if loopVar > dst then error("Overflow") end
+   end
+  else
+   error("unable to handle cl instruction " .. instr)
+  end
+ end
+ distlens[-1] = nil
+end
+
+function $dfReadHuffmanDynamic()
+ local metalensi = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
+ local metalens = {}
+ for loopVar = 0, 18 do metalens[loopVar] = 0 end
+ local ltl = $dfGetIntField(5) + 257
+ local dst = $dfGetIntField(5) + 1
+ local cln = $dfGetIntField(4) + 4
+ for loopVar = 1, cln do
+  metalens[metalensi[loopVar]] = $dfGetIntField(3)
+ end
+ local metatree = $dfGenHuffmanTree(metalens)
+ local alllens = {}
+ $dfReadHuffmanDynamicSubcodes(alllens, ltl + dst, metatree)
+ local litlens = {}
+ local distlens = {}
+ for loopVar = 0, ltl - 1 do
+  litlens[loopVar] = alllens[loopVar]
+ end
+ for loopVar = 0, dst - 1 do
+  distlens[loopVar] = alllens[ltl + loopVar]
+ end
+ return $dfGenHuffmanTree(litlens), $dfGenHuffmanTree(distlens)
+end
+
+-- Main Thread --
+
+$dfThread = coroutine.create(function ($a0, $a1)
+ while true do
+  $a0 = coroutine.yield()
+  $NTdfBlockType
+  $dfBlockType = $dfGetIntField(2)
+  if $dfBlockType == 0 then
+   -- literal
+   $dfAlignToByte()
+   $a1 = $dfGetIntField(16)
+   -- this is weird, ignore it
+   $dfGetIntField(16)
+   for loopVar = 1, $a1 do
+    $dfOutput(string.char($dfGetIntField(8)))
+   end
+  elseif $dfBlockType == 1 then
+   -- fixed Huffman
+   $dfReadBlockBody($dfFixedLit, $dfFixedDst)
+  elseif $dfBlockType == 2 then
+   -- dynamic Huffman
+   $dfReadBlockBody($dfReadHuffmanDynamic())
+  else
+   error("b3")
+  end
+  $DTdfBlockType
+  while $a0 do
+   coroutine.yield()
+  end
+ end
+end)
+
+-- The Outer Engine --
+
+coroutine.resume($dfThread)
+function $engineInput($a0, $a1)
+ $NTdfForLoopVar
+ $NTdfForLoopVar2
+ for $dfForLoopVar = 1, #$a0 do
+  $a1 = $a0:byte($dfForLoopVar)
+  $dfAlignToByteRemaining = 8
+  while $dfAlignToByteRemaining > 0 do
+   -- If we're providing the first bit (v = 8), then there are 7 bits remaining.
+   -- So this hits 0 when the *next* 8 yields will provide an as-is byte.
+   $dfAlignToByteRemaining = $dfAlignToByteRemaining - 1
+   assert(coroutine.resume($dfThread, $a1 % 2 == 1))
+   $a1 = math.floor($a1 / 2)
+  end
+ end
+ $DTdfForLoopVar2
+ $DTdfForLoopVar
+ -- flush prepared buffer
+ $engineOutput($dfPushBuf)
+ $dfPushBuf = ""
+end
+

inst/instbase.lua

@@ -1,139 +0,0 @@
--- KOSNEO installer base
--- This is released into the public domain.
--- No warranty is provided, implied or otherwise.
-
--- DECOMPRESSION ENGINE PRECEDES THIS CODE --
--- NOTE: upper-case is reserved for this file,
--- lower-case is reserved for the decompression engine
-
--- A: temporary
-
--- B: computer
--- C: component
-
--- D: additional temporary
--- E: read-in state machine
-
--- F: current file: filename
--- H: remaining bytes to copy/skip
--- I: current file: handle (nil if not writing)
-
--- J: sectors handled
--- K: sector count (injected during build)
--- L: compression engine data function (set by CE)
--- M: sector handler function (called by CE)
-
--- O: current character for read-in state machine
--- P: file handle for selfread
-
--- Q: octal decoding function
-
--- X: screen address
--- Y: component: gpu
--- Z: component: filesystem
-B = computer
-C = component
-assert(C, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")
-
-X = C.list("screen", true)()
-Y = C.list("gpu", true)()
-
-Z = C.proxy(B.getBootAddress())
-
-Z.remove("init.neoi.lua")
-Z.rename("init.lua","init.neoi.lua")
-P = Z.open("init.neoi.lua","rb")
-
-F = "Starting..."
-H = 0
-
-if X and Y then
- Y = C.proxy(Y)
- Y.bind(X)
- Y.setResolution(50, 5)
- Y.setBackground(2^24-1)
- Y.setForeground(0)
- Y.fill(1, 1, 50, 5, "█")
- Y.fill(1, 2, 50, 1, " ")
- Y.set(2, 2, "KittenOS NEO Installer")
-end
-
-function Q(A)
- if A == "" then return 0 end
- return Q(A:sub(1, -2)) * 8 + (A:byte(#A) - 48)
-end
-
-J = 0
-
-function M(n)
- if H > 0 then
-  A = math.min(512, H)
-  H = H - A
-  if I then
-   Z.write(I, n:sub(1, A))
-   if H <= 0 then
-    Z.close(I)
-    I = nil
-   end
-  end
- else
-  F = n:sub(1, 100):gsub("\x00", "")
-  -- this sets up the reading/skipping of data
-  H = Q(n:sub(125, 135))
-  if F:sub(1, 2) == "./" and F ~= "./" then
-   F = F:sub(3)
-   if F:sub(#F) == "/" then
-    Z.makeDirectory(F)
-   else
-    I = Z.open(F, "wb")
-    if H == 0 then
-     Z.close(I)
-     I = nil
-    end
-   end
-  end
- end
- -- UPDATE DISPLAY --
- J = J + 1
- if X and Y then
-  Y.fill(1, 2, 50, 1, " ")
-  Y.set(2, 2, "KittenOS NEO Installer : " .. F)
-  Y.fill(2, 4, 48, 1, "█")
-  Y.fill(2, 4, math.ceil(48 * J / K), 1, " ")
- end
- if J % 8 == 0 then
-  B.pullSignal(0.01)
- end
- if J == K then
-  Z.close(P)
-  Z.remove("init.neoi.lua")
-  B.shutdown(true)
- end
-end
-
-while true do
- A = Z.read(P, 64)
- D = ""
- for i = 1, #A do
-  -- Read-in state machine
-  O = A:sub(i, i)
-  if not E then
-   if O == "\x00" then
-    E = 0
-   end
-  elseif E == 0 then
-   if O == "\xFE" then
-    E = 1
-   else
-    D = D .. O
-   end
-  else
-   D = D .. O
-   E = 0
-  end
- end
- L(D)
-end
-
--- COMPRESSED DATA FOLLOWS THIS CODE --
-

inst/instcore.lua

@@ -0,0 +1,76 @@
+-- KOSNEO installer base
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+$icScreen = $component.list("screen", true)()
+$icGPU = $component.list("gpu", true)()
+
+$icFilename = "Starting..."
+$icBytesRemaining = 0
+
+if $icScreen and $icGPU then
+ $icGPU = $component.proxy($icGPU)
+ $icGPU.bind($icScreen)
+ $icGPU.setResolution(50, 5)
+ $icGPU.setBackground(2^24-1)
+ $icGPU.setForeground(0)
+ $icGPU.fill(1, 1, 50, 5, "█")
+ $icGPU.fill(1, 2, 50, 1, " ")
+ $icGPU.set(2, 2, "KittenOS NEO Installer")
+end
+
+function $icOctalToNumber($a0)
+ if $a0 == "" then return 0 end
+ return $icOctalToNumber($a0:sub(1, -2)) * 8 + ($a0:byte(#$a0) - 48)
+end
+
+$icSectorsRead = 0
+$iBlockingLen = 512
+function $iBlockingHook($a0)
+ if $icBytesRemaining > 0 then
+  $NTicByteAdv
+  $icByteAdv = math.min(512, $icBytesRemaining)
+  $icBytesRemaining = $icBytesRemaining - $icByteAdv
+  if $icFile then
+   $filesystem.write($icFile, $a0:sub(1, $icByteAdv))
+   if $icBytesRemaining <= 0 then
+    $filesystem.close($icFile)
+    $icFile = nil
+   end
+  end
+  $DTicByteAdv
+ else
+  $icFilename = $a0:sub(1, 100):gsub("\x00", "")
+  -- this sets up the reading/skipping of data
+  $icBytesRemaining = $icOctalToNumber($a0:sub(125, 135))
+  if $icFilename:sub(1, 2) == "./" and $icFilename ~= "./" then
+   $icFilename = $icFilename:sub(3)
+   if $icFilename:sub(#$icFilename) == "/" then
+    $filesystem.makeDirectory($icFilename)
+   else
+    $icFile = $filesystem.open($icFilename, "wb")
+    if $icBytesRemaining == 0 then
+     $filesystem.close($icFile)
+     $icFile = nil
+    end
+   end
+  end
+ end
+ -- UPDATE DISPLAY --
+ $icSectorsRead = $icSectorsRead + 1
+ if $icScreen and $icGPU then
+  $icGPU.fill(1, 2, 50, 1, " ")
+  $icGPU.set(2, 2, "KittenOS NEO Installer : " .. $icFilename)
+  $icGPU.fill(2, 4, 48, 1, "█")
+  $icGPU.fill(2, 4, math.ceil(48 * $icSectorsRead / $$SECTORS), 1, " ")
+ end
+ if $icSectorsRead % 16 == 0 then
+  $computer.pullSignal(0.01)
+ end
+ if $icSectorsRead == $$SECTORS then
+  $filesystem.close($readInFile)
+  $filesystem.remove("init.neoi.lua")
+  $computer.shutdown(true)
+ end
+end
+

inst/insthead.lua

@@ -0,0 +1,34 @@
+-- KOSNEO installer base
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+$computer = computer
+$component = component
+assert($component, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")
+
+$filesystem = $component.proxy($computer.getBootAddress())
+
+$filesystem.remove("init.neoi.lua")
+$filesystem.rename("init.lua", "init.neoi.lua")
+$readInFile = $filesystem.open("init.neoi.lua", "rb")
+
+$iBlockingBuffer = ""
+$iBlockingLen = $$CORESIZE
+$iBlockingHook = function ($a0)
+ -- This takes over the iBlockingHook.
+ assert(load($a0))()
+end
+
+$engineOutput = function ($a0)
+ $iBlockingBuffer = $iBlockingBuffer .. $a0
+ while #$iBlockingBuffer >= $iBlockingLen do
+  $NTiBlock
+  $iBlock = $iBlockingBuffer:sub(1, $iBlockingLen)
+  $iBlockingBuffer = $iBlockingBuffer:sub($iBlockingLen + 1)
+  $iBlockingHook($iBlock)
+  $DTiBlock
+ end
+end
+
+-- DECOMPRESSION ENGINE FOLLOWS THIS CODE --
+

inst/insttail.lua

@@ -0,0 +1,32 @@
+-- KOSNEO installer base
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- DECOMPRESSION ENGINE PRECEDES THIS CODE --
+
+while true do
+ $readInBlock = $filesystem.read($readInFile, 1024)
+ for i = 1, #$readInBlock do
+  -- Read-in state machine
+  $NTreadInChar
+  $readInChar = $readInBlock:sub(i, i)
+  if not $readInState then
+   if $readInChar == "\x00" then
+    $readInState = 0
+   end
+  elseif $readInState == 0 then
+   if $readInChar == "\xFE" then
+    $readInState = 1
+   else
+    $engineInput($readInChar)
+   end
+  else
+   $engineInput($readInChar)
+   $readInState = 0
+  end
+ end
+ $DTreadInChar
+end
+
+-- COMPRESSED DATA FOLLOWS THIS CODE --
+

inst/libs/lexcrunch.lua

@@ -72,23 +72,77 @@ local function pass(buffer)
  return ob
 end
 
-return function (op)
- -- comment removal
- while true do
-  local np = op:gsub("%-%-[^\n]*\n", " ")
-  np = np:gsub("%-%-[^\n]*$", "")
-  if np == op then
-   break
-  end
-  op = np
+-- Context creation --
+return function ()
+ local forwardSymTab = {}
+ local reverseSymTab = {}
+
+ local temporaryPool = {}
+
+ local possible = {}
+ for i = 1, 52 do
+  possible[i] = ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"):sub(i, i)
  end
- -- stripping
- while true do
-  local np = pass(op)
-  if np == op then
-   return np
+
+ local function allocate(reason)
+  for _, v in pairs(possible) do
+   if not reverseSymTab[v] then
+    reverseSymTab[v] = reason
+    return v
+   end
   end
-  op = np
  end
- return op
+
+ return function (op, defines)
+  -- symbol replacement
+  op = op:gsub("%$[%$a-zA-Z0-9]*", function (str)
+   if str:sub(2, 2) == "$" then
+    -- defines
+    assert(defines[str], "no define " .. str)
+    return defines[str]
+   elseif str:sub(2, 3) == "NT" then
+    -- temporaries +
+    local id = "$" .. str:sub(4)
+    assert(not forwardSymTab[id], "var already exists: " .. id)
+    local val = table.remove(temporaryPool, 1)
+    if not val then val = allocate("temporary") end
+    forwardSymTab[id] = val
+    return ""
+   elseif str:sub(2, 3) == "DT" then
+    -- temporaries -
+    local id = "$" .. str:sub(4)
+    assert(forwardSymTab[id], "no such var: " .. id)
+    assert(reverseSymTab[forwardSymTab[id]] == "temporary", "var not allocated as temporary: " .. id)
+    table.insert(temporaryPool, forwardSymTab[id])
+    forwardSymTab[id] = nil
+    return ""
+   else
+    -- normal handling
+    if forwardSymTab[str] then
+     return forwardSymTab[str]
+    end
+    local v = allocate(str)
+    forwardSymTab[str] = v
+    return v
+   end
+  end)
+  -- comment removal
+  while true do
+   local np = op:gsub("%-%-[^\n]*\n", " ")
+   np = np:gsub("%-%-[^\n]*$", "")
+   if np == op then
+    break
+   end
+   op = np
+  end
+  -- stripping
+  while true do
+   local np = pass(op)
+   if np == op then
+    return np
+   end
+   op = np
+  end
+  return op
+ end
 end

inst/status.lua

@@ -0,0 +1,36 @@
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- Status Screen --
+local target = ...
+local u = require("libs.frw")
+local instSize = #u.read(target)
+
+local status = ""
+local statusDetail = ""
+local blinkI = ""
+if instSize > 65536 then
+ blinkI = "5;31;"
+ status = " DO NOT SHIP "
+ statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
+elseif instSize > 64000 then
+ blinkI = "33;"
+ status = " Shippable * "
+ statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
+else
+ blinkI = "32;"
+ status = "  All Green  "
+ statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
+end
+io.stderr:write("\n")
+local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
+io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
+io.stderr:write(ctS .. status          .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
+io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
+io.stderr:write("\n")
+io.stderr:write(statusDetail .. "\n")
+io.stderr:write("\n")
+io.stderr:write("Size:  " .. instSize .. "\n")
+io.stderr:write(" max.  65536\n")
+io.stderr:write("\n")
+

inst/uncompressed/compress.lua

@@ -1,7 +1,10 @@
 -- This is released into the public domain.
 -- No warranty is provided, implied or otherwise.
 
-return function (b)
- return b
+-- Example compression engine.
+-- Given: data, lexCrunch
+--  returns compressionEngine, compressedData
+return function (data, lexCrunch)
+ return lexCrunch(" $engineInput = $engineOutput ", {}), data
 end
 

inst/uncompressed/instdeco.lua

@@ -1,13 +0,0 @@
--- This is released into the public domain.
--- No warranty is provided, implied or otherwise.
-
-t = ""
-function L(d)
- if not d then return end
- t = t .. d
- while #t >= 512 do
-  M(t:sub(1, 512))
-  t = t:sub(513)
- end
-end
-

inst/verify.lua

@@ -1,30 +0,0 @@
--- This is released into the public domain.
--- No warranty is provided, implied or otherwise.
-
--- Installer Compression Verification Tool --
-local alg, tarName = ...
-local u = require("libs.frw")
-
-io.stderr:write("verifying... ")
-local p = u.progress()
-
-local tarData = u.read(tarName)
-
-local total = ""
-function M(t)
- assert(#t == 512)
- total = total .. t
- p(#total / #tarData)
-end
-
-dofile(alg .. "/instdeco.lua")
-
-L(u.read(alg .. "/output.bin"))
-
-if total ~= tarData then
- io.stderr:write("\n" .. #total .. " : " .. #tarData .. "\n")
- u.write(alg .. "/vfyerr.bin", total)
- error("VERIFICATION FAILURE : see inst/" .. alg .. "/vfyerr.bin!")
-end
-io.stderr:write("\nverification success\n")
-

package.sh

@@ -16,7 +16,7 @@ cd ..
 # The Installer Creator
 cd inst
 lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua
-lua verify.lua $1 ../code.tar
+lua status.lua ../inst.lua
 cd ..
 
 # Common Repository Setup Code