All the architectural changes pre-DEFLATE

inst/bdivide/compress.lua

@@ -105,7 +105,7 @@ local function bdivide(blk, p)
  return out
 end
 
-return function (data)
+return function (data, lexCrunch)
  io.stderr:write("preproc: ")
  local pi = frw.progress()
  local function p(b)
@@ -120,5 +120,5 @@ return function (data)
  -- It's cheaper than the required code.
  -- 1 byte of buffer for preproc,
  -- 2 bytes of buffer for bdivide.
- return data .. ("\x00"):rep(3)
+ return lexCrunch(frw.read("bdivide/instdeco.lua"), {}), data .. ("\x00"):rep(3)
 end

inst/bdivide/instdeco.lua

@@ -4,23 +4,11 @@
 -- BDIVIDE (r5 edition) and PREPROC (r9 edition)
 -- decompression engine for installer
 
--- a: temporary
+$bdPPBuffer = ""
---    touched by q,L
+$bdBDBuffer = ""
--- b: sector accumulator
+$bdBDWindow = ("\x00"):rep(2^16)
--- c: bdivide accumulator
--- d: temporary
---    touched by q,L
--- t: preproc accumulator
--- q: function to submit to preproc
--- s: temporary
---    touched by L
--- w: bdivide window
-
--- L: function to submit to bdivide
-
-b,t,c,w="","","",("\x00"):rep(2^16)
 -- High-level breakdown:
--- q is unescaper & TAR-sector-breakup.
+-- q is unescaper.
 -- It'll only begin to input if at least 3 bytes are available,
 --  so you'll want to throw in 2 extra zeroes at the end of stream as done here.
 -- It uses t (input buffer) and p (output buffer).
@@ -31,7 +19,7 @@ b,t,c,w="","","",("\x00"):rep(2^16)
 -- And it also uses a fake local.
 
 -- SEE compress.lua FOR THIS FUNCTION
-function p(x, y)
+function $bdPP(x, y)
  if x == 126 then
   return string.char(y), 3
  elseif x == 127 then
@@ -46,31 +34,29 @@ function p(x, y)
  return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2
 end
 
-function q(w)
+function $engineInput($a0)
- t = t .. w
+ $bdBDBuffer = $bdBDBuffer .. $a0
- while #t > 1 do
+ while #$bdBDBuffer > 2 do
-  d, a = p(t:byte(), t:byte(2))
+  $a0 = $bdBDBuffer:byte()
-  b = b .. d
+  if $a0 < 128 then
-  t = t:sub(a)
+   $a0 = $bdBDBuffer:sub(1, 1)
-  if #b > 511 then
+   $bdBDBuffer = $bdBDBuffer:sub(2)
-   M(b:sub(1, 512))
-   b = b:sub(513)
-  end
- end
-end
-
-function L(d)
- c = c .. d
- while #c > 2 do
-  s = c:byte()
-  if s < 128 then
-   s, c = c:sub(1, 1), c:sub(2)
   else
-   a = c:byte(2) * 256 + c:byte(3) + 1
+   $NTbdBDPtr
-   s, c = w:sub(a, a + s - 125), c:sub(4)
+   $bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1
+   $a0 = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $a0 - 125)
+   $bdBDBuffer = $bdBDBuffer:sub(4)
+   $DTbdBDPtr
+  end
+  $bdPPBuffer = $bdPPBuffer .. $a0
+  $bdBDWindow = ($bdBDWindow .. $a0):sub(-2^16)
+  while #$bdPPBuffer > 1 do
+   $NTbdPPAdv
+   $a0, $bdPPAdv = $bdPP($bdPPBuffer:byte(), $bdPPBuffer:byte(2))
+   $bdPPBuffer = $bdPPBuffer:sub($bdPPAdv)
+   $DTbdPPAdv
+   $engineOutput($a0)
   end
-  q(s)
-  w = (w .. s):sub(-2^16)
  end
 end
 

inst/build.lua

@@ -7,8 +7,7 @@ cid = (cid or "UNKNOWN"):sub(1, 7)
 
 local u = require("libs.frw")
 
-local tarData = u.read(tarName)
+local algImpl = require(alg .. ".compress")
-local tarSectors = math.floor(#tarData / 512)
 
 local instSize = 0
 local function put(data)
@@ -16,52 +15,31 @@ local function put(data)
  instSize = instSize + #data
 end
 
-put("--" .. cid .. "\n")
+-- TAR File --
-put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
+local tarData = u.read(tarName)
+local tarSectors = math.floor(#tarData / 512)
 
-local instCode = "K=" .. tarSectors .. "\n" .. u.read(alg .. "/instdeco.lua") .. u.read("instbase.lua")
+-- Installer Lexcrunch Context --
-instCode = require("libs.lexcrunch")(instCode)
+local lexCrunch = require("libs.lexcrunch")()
-put(instCode)
 
--- the \x00 is the indicator to start reading
+local installerCore = lexCrunch(u.read("instcore.lua"), {["$$SECTORS"] = tostring(tarSectors)})
-put("--[[\x00")
+local installerHead = lexCrunch(u.read("insthead.lua"), {["$$CORESIZE"] = tostring(#installerCore)})
+local installerTail = lexCrunch(u.read("insttail.lua"), {})
 
+-- Installer Compression --
+local rawData = installerCore .. tarData
 io.stderr:write("compressing...\n")
-local compressedData = require(alg .. ".compress")(tarData)
+local compressionEngine, compressedData = algImpl(rawData, lexCrunch)
-u.write(alg .. "/output.bin", compressedData)
+-- RISM [[
-io.stderr:write("compression with " .. alg .. ": " .. #tarData .. " -> " .. #compressedData .. "\n")
--- Program the read-in state machine
 compressedData = compressedData:gsub("\xFE", "\xFE\xFE")
 compressedData = compressedData:gsub("]]", "]\xFE]")
+compressedData = "\x00" .. compressedData
+-- ]]
+io.stderr:write("compression with " .. alg .. ": " .. #rawData .. " -> " .. #compressedData .. "\n")
 
-put(compressedData)
+-- Installer Final Generation --
-put("]]")
+put("--" .. cid .. "\n")
-
+put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
-local status = ""
+put(lexCrunch(installerHead .. compressionEngine .. installerTail, {}))
-local statusDetail = ""
+put("--[[" .. compressedData .. "]]")
-local blinkI = ""
-if instSize > 65536 then
- blinkI = "5;31;"
- status = " DO NOT SHIP "
- statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
-elseif instSize > 64000 then
- blinkI = "33;"
- status = " Shippable * "
- statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
-else
- blinkI = "32;"
- status = "  All Green  "
- statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
-end
-io.stderr:write("\n")
-local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
-io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
-io.stderr:write(ctS .. status          .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
-io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
-io.stderr:write("\n")
-io.stderr:write(statusDetail .. "\n")
-io.stderr:write("\n")
-io.stderr:write("Size:  " .. instSize .. "\n")
-io.stderr:write(" max.  65536\n")
-io.stderr:write("\n")
 

inst/instbase.lua

@@ -1,139 +0,0 @@
--- KOSNEO installer base
--- This is released into the public domain.
--- No warranty is provided, implied or otherwise.
-
--- DECOMPRESSION ENGINE PRECEDES THIS CODE --
--- NOTE: upper-case is reserved for this file,
--- lower-case is reserved for the decompression engine
-
--- A: temporary
-
--- B: computer
--- C: component
-
--- D: additional temporary
--- E: read-in state machine
-
--- F: current file: filename
--- H: remaining bytes to copy/skip
--- I: current file: handle (nil if not writing)
-
--- J: sectors handled
--- K: sector count (injected during build)
--- L: compression engine data function (set by CE)
--- M: sector handler function (called by CE)
-
--- O: current character for read-in state machine
--- P: file handle for selfread
-
--- Q: octal decoding function
-
--- X: screen address
--- Y: component: gpu
--- Z: component: filesystem
-B = computer
-C = component
-assert(C, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")
-
-X = C.list("screen", true)()
-Y = C.list("gpu", true)()
-
-Z = C.proxy(B.getBootAddress())
-
-Z.remove("init.neoi.lua")
-Z.rename("init.lua","init.neoi.lua")
-P = Z.open("init.neoi.lua","rb")
-
-F = "Starting..."
-H = 0
-
-if X and Y then
- Y = C.proxy(Y)
- Y.bind(X)
- Y.setResolution(50, 5)
- Y.setBackground(2^24-1)
- Y.setForeground(0)
- Y.fill(1, 1, 50, 5, "█")
- Y.fill(1, 2, 50, 1, " ")
- Y.set(2, 2, "KittenOS NEO Installer")
-end
-
-function Q(A)
- if A == "" then return 0 end
- return Q(A:sub(1, -2)) * 8 + (A:byte(#A) - 48)
-end
-
-J = 0
-
-function M(n)
- if H > 0 then
-  A = math.min(512, H)
-  H = H - A
-  if I then
-   Z.write(I, n:sub(1, A))
-   if H <= 0 then
-    Z.close(I)
-    I = nil
-   end
-  end
- else
-  F = n:sub(1, 100):gsub("\x00", "")
-  -- this sets up the reading/skipping of data
-  H = Q(n:sub(125, 135))
-  if F:sub(1, 2) == "./" and F ~= "./" then
-   F = F:sub(3)
-   if F:sub(#F) == "/" then
-    Z.makeDirectory(F)
-   else
-    I = Z.open(F, "wb")
-    if H == 0 then
-     Z.close(I)
-     I = nil
-    end
-   end
-  end
- end
- -- UPDATE DISPLAY --
- J = J + 1
- if X and Y then
-  Y.fill(1, 2, 50, 1, " ")
-  Y.set(2, 2, "KittenOS NEO Installer : " .. F)
-  Y.fill(2, 4, 48, 1, "█")
-  Y.fill(2, 4, math.ceil(48 * J / K), 1, " ")
- end
- if J % 8 == 0 then
-  B.pullSignal(0.01)
- end
- if J == K then
-  Z.close(P)
-  Z.remove("init.neoi.lua")
-  B.shutdown(true)
- end
-end
-
-while true do
- A = Z.read(P, 64)
- D = ""
- for i = 1, #A do
-  -- Read-in state machine
-  O = A:sub(i, i)
-  if not E then
-   if O == "\x00" then
-    E = 0
-   end
-  elseif E == 0 then
-   if O == "\xFE" then
-    E = 1
-   else
-    D = D .. O
-   end
-  else
-   D = D .. O
-   E = 0
-  end
- end
- L(D)
-end
-
--- COMPRESSED DATA FOLLOWS THIS CODE --
-

inst/instcore.lua

@@ -0,0 +1,76 @@
+-- KOSNEO installer base
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+$icScreen = $component.list("screen", true)()
+$icGPU = $component.list("gpu", true)()
+
+$icFilename = "Starting..."
+$icBytesRemaining = 0
+
+if $icScreen and $icGPU then
+ $icGPU = $component.proxy($icGPU)
+ $icGPU.bind($icScreen)
+ $icGPU.setResolution(50, 5)
+ $icGPU.setBackground(2^24-1)
+ $icGPU.setForeground(0)
+ $icGPU.fill(1, 1, 50, 5, "█")
+ $icGPU.fill(1, 2, 50, 1, " ")
+ $icGPU.set(2, 2, "KittenOS NEO Installer")
+end
+
+function $icOctalToNumber($a0)
+ if $a0 == "" then return 0 end
+ return $icOctalToNumber($a0:sub(1, -2)) * 8 + ($a0:byte(#$a0) - 48)
+end
+
+$icSectorsRead = 0
+$iBlockingLen = 512
+function $iBlockingHook($a0)
+ if $icBytesRemaining > 0 then
+  $NTicByteAdv
+  $icByteAdv = math.min(512, $icBytesRemaining)
+  $icBytesRemaining = $icBytesRemaining - $icByteAdv
+  if $icFile then
+   $filesystem.write($icFile, $a0:sub(1, $icByteAdv))
+   if $icBytesRemaining <= 0 then
+    $filesystem.close($icFile)
+    $icFile = nil
+   end
+  end
+  $DTicByteAdv
+ else
+  $icFilename = $a0:sub(1, 100):gsub("\x00", "")
+  -- this sets up the reading/skipping of data
+  $icBytesRemaining = $icOctalToNumber($a0:sub(125, 135))
+  if $icFilename:sub(1, 2) == "./" and $icFilename ~= "./" then
+   $icFilename = $icFilename:sub(3)
+   if $icFilename:sub(#$icFilename) == "/" then
+    $filesystem.makeDirectory($icFilename)
+   else
+    $icFile = $filesystem.open($icFilename, "wb")
+    if $icBytesRemaining == 0 then
+     $filesystem.close($icFile)
+     $icFile = nil
+    end
+   end
+  end
+ end
+ -- UPDATE DISPLAY --
+ $icSectorsRead = $icSectorsRead + 1
+ if $icScreen and $icGPU then
+  $icGPU.fill(1, 2, 50, 1, " ")
+  $icGPU.set(2, 2, "KittenOS NEO Installer : " .. $icFilename)
+  $icGPU.fill(2, 4, 48, 1, "█")
+  $icGPU.fill(2, 4, math.ceil(48 * $icSectorsRead / $$SECTORS), 1, " ")
+ end
+ if $icSectorsRead % 16 == 0 then
+  $computer.pullSignal(0.01)
+ end
+ if $icSectorsRead == $$SECTORS then
+  $filesystem.close($readInFile)
+  $filesystem.remove("init.neoi.lua")
+  $computer.shutdown(true)
+ end
+end
+

inst/insthead.lua

@@ -0,0 +1,34 @@
+-- KOSNEO installer base
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+$computer = computer
+$component = component
+assert($component, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")
+
+$filesystem = $component.proxy($computer.getBootAddress())
+
+$filesystem.remove("init.neoi.lua")
+$filesystem.rename("init.lua", "init.neoi.lua")
+$readInFile = $filesystem.open("init.neoi.lua", "rb")
+
+$iBlockingBuffer = ""
+$iBlockingLen = $$CORESIZE
+$iBlockingHook = function ($a0)
+ -- This takes over the iBlockingHook.
+ assert(load($a0))()
+end
+
+$engineOutput = function ($a0)
+ $iBlockingBuffer = $iBlockingBuffer .. $a0
+ while #$iBlockingBuffer >= $iBlockingLen do
+  $NTiBlock
+  $iBlock = $iBlockingBuffer:sub(1, $iBlockingLen)
+  $iBlockingBuffer = $iBlockingBuffer:sub($iBlockingLen + 1)
+  $iBlockingHook($iBlock)
+  $DTiBlock
+ end
+end
+
+-- DECOMPRESSION ENGINE FOLLOWS THIS CODE --
+

inst/insttail.lua

@@ -0,0 +1,32 @@
+-- KOSNEO installer base
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- DECOMPRESSION ENGINE PRECEDES THIS CODE --
+
+while true do
+ $readInBlock = $filesystem.read($readInFile, 1024)
+ for i = 1, #$readInBlock do
+  -- Read-in state machine
+  $NTreadInChar
+  $readInChar = $readInBlock:sub(i, i)
+  if not $readInState then
+   if $readInChar == "\x00" then
+    $readInState = 0
+   end
+  elseif $readInState == 0 then
+   if $readInChar == "\xFE" then
+    $readInState = 1
+   else
+    $engineInput($readInChar)
+   end
+  else
+   $engineInput($readInChar)
+   $readInState = 0
+  end
+ end
+ $DTreadInChar
+end
+
+-- COMPRESSED DATA FOLLOWS THIS CODE --
+

inst/libs/lexcrunch.lua

@@ -72,23 +72,77 @@ local function pass(buffer)
  return ob
 end
 
-return function (op)
+-- Context creation --
- -- comment removal
+return function ()
- while true do
+ local forwardSymTab = {}
-  local np = op:gsub("%-%-[^\n]*\n", " ")
+ local reverseSymTab = {}
-  np = np:gsub("%-%-[^\n]*$", "")
+
-  if np == op then
+ local temporaryPool = {}
-   break
+
-  end
+ local possible = {}
-  op = np
+ for i = 1, 52 do
+  possible[i] = ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"):sub(i, i)
  end
- -- stripping
+
- while true do
+ local function allocate(reason)
-  local np = pass(op)
+  for _, v in pairs(possible) do
-  if np == op then
+   if not reverseSymTab[v] then
-   return np
+    reverseSymTab[v] = reason
+    return v
+   end
   end
-  op = np
  end
- return op
+
+ return function (op, defines)
+  -- symbol replacement
+  op = op:gsub("%$[%$a-zA-Z0-9]*", function (str)
+   if str:sub(2, 2) == "$" then
+    -- defines
+    assert(defines[str], "no define " .. str)
+    return defines[str]
+   elseif str:sub(2, 3) == "NT" then
+    -- temporaries +
+    local id = "$" .. str:sub(4)
+    assert(not forwardSymTab[id], "var already exists: " .. id)
+    local val = table.remove(temporaryPool, 1)
+    if not val then val = allocate("temporary") end
+    forwardSymTab[id] = val
+    return ""
+   elseif str:sub(2, 3) == "DT" then
+    -- temporaries -
+    local id = "$" .. str:sub(4)
+    assert(forwardSymTab[id], "no such var: " .. id)
+    assert(reverseSymTab[forwardSymTab[id]] == "temporary", "var not allocated as temporary: " .. id)
+    table.insert(temporaryPool, forwardSymTab[id])
+    forwardSymTab[id] = nil
+    return ""
+   else
+    -- normal handling
+    if forwardSymTab[str] then
+     return forwardSymTab[str]
+    end
+    local v = allocate(str)
+    forwardSymTab[str] = v
+    return v
+   end
+  end)
+  -- comment removal
+  while true do
+   local np = op:gsub("%-%-[^\n]*\n", " ")
+   np = np:gsub("%-%-[^\n]*$", "")
+   if np == op then
+    break
+   end
+   op = np
+  end
+  -- stripping
+  while true do
+   local np = pass(op)
+   if np == op then
+    return np
+   end
+   op = np
+  end
+  return op
+ end
 end

inst/status.lua

@@ -0,0 +1,36 @@
+-- This is released into the public domain.
+-- No warranty is provided, implied or otherwise.
+
+-- Status Screen --
+local target = ...
+local u = require("libs.frw")
+local instSize = #u.read(target)
+
+local status = ""
+local statusDetail = ""
+local blinkI = ""
+if instSize > 65536 then
+ blinkI = "5;31;"
+ status = " DO NOT SHIP "
+ statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
+elseif instSize > 64000 then
+ blinkI = "33;"
+ status = " Shippable * "
+ statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
+else
+ blinkI = "32;"
+ status = "  All Green  "
+ statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
+end
+io.stderr:write("\n")
+local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
+io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
+io.stderr:write(ctS .. status          .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
+io.stderr:write(ctS .. "             " .. ctM .. "         " .. ctE)
+io.stderr:write("\n")
+io.stderr:write(statusDetail .. "\n")
+io.stderr:write("\n")
+io.stderr:write("Size:  " .. instSize .. "\n")
+io.stderr:write(" max.  65536\n")
+io.stderr:write("\n")
+

inst/uncompressed/compress.lua

@@ -1,7 +1,10 @@
 -- This is released into the public domain.
 -- No warranty is provided, implied or otherwise.
 
-return function (b)
+-- Example compression engine.
- return b
+-- Given: data, lexCrunch
+--  returns compressionEngine, compressedData
+return function (data, lexCrunch)
+ return lexCrunch(" $engineInput = $engineOutput ", {}), data
 end
 

inst/uncompressed/instdeco.lua

@@ -1,13 +0,0 @@
--- This is released into the public domain.
--- No warranty is provided, implied or otherwise.
-
-t = ""
-function L(d)
- if not d then return end
- t = t .. d
- while #t >= 512 do
-  M(t:sub(1, 512))
-  t = t:sub(513)
- end
-end
-

inst/verify.lua

@@ -1,30 +0,0 @@
--- This is released into the public domain.
--- No warranty is provided, implied or otherwise.
-
--- Installer Compression Verification Tool --
-local alg, tarName = ...
-local u = require("libs.frw")
-
-io.stderr:write("verifying... ")
-local p = u.progress()
-
-local tarData = u.read(tarName)
-
-local total = ""
-function M(t)
- assert(#t == 512)
- total = total .. t
- p(#total / #tarData)
-end
-
-dofile(alg .. "/instdeco.lua")
-
-L(u.read(alg .. "/output.bin"))
-
-if total ~= tarData then
- io.stderr:write("\n" .. #total .. " : " .. #tarData .. "\n")
- u.write(alg .. "/vfyerr.bin", total)
- error("VERIFICATION FAILURE : see inst/" .. alg .. "/vfyerr.bin!")
-end
-io.stderr:write("\nverification success\n")
-

package.sh

@@ -16,7 +16,7 @@ cd ..
 # The Installer Creator
 cd inst
 lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua
-lua verify.lua $1 ../code.tar
+lua status.lua ../inst.lua
 cd ..
 
 # Common Repository Setup Code