All the architectural changes pre-DEFLATE

This commit is contained in:
20kdc 2020-03-31 11:59:58 +01:00
parent dd21abe8fa
commit bc4d626b4e
13 changed files with 303 additions and 286 deletions

View File

@ -105,7 +105,7 @@ local function bdivide(blk, p)
return out
end
return function (data)
return function (data, lexCrunch)
io.stderr:write("preproc: ")
local pi = frw.progress()
local function p(b)
@ -120,5 +120,5 @@ return function (data)
-- It's cheaper than the required code.
-- 1 byte of buffer for preproc,
-- 2 bytes of buffer for bdivide.
return data .. ("\x00"):rep(3)
return lexCrunch(frw.read("bdivide/instdeco.lua"), {}), data .. ("\x00"):rep(3)
end

View File

@ -4,23 +4,11 @@
-- BDIVIDE (r5 edition) and PREPROC (r9 edition)
-- decompression engine for installer
-- a: temporary
-- touched by q,L
-- b: sector accumulator
-- c: bdivide accumulator
-- d: temporary
-- touched by q,L
-- t: preproc accumulator
-- q: function to submit to preproc
-- s: temporary
-- touched by L
-- w: bdivide window
-- L: function to submit to bdivide
b,t,c,w="","","",("\x00"):rep(2^16)
$bdPPBuffer = ""
$bdBDBuffer = ""
$bdBDWindow = ("\x00"):rep(2^16)
-- High-level breakdown:
-- q is unescaper & TAR-sector-breakup.
-- q is unescaper.
-- It'll only begin to input if at least 3 bytes are available,
-- so you'll want to throw in 2 extra zeroes at the end of stream as done here.
-- It uses t (input buffer) and p (output buffer).
@ -31,7 +19,7 @@ b,t,c,w="","","",("\x00"):rep(2^16)
-- And it also uses a fake local.
-- SEE compress.lua FOR THIS FUNCTION
function p(x, y)
function $bdPP(x, y)
if x == 126 then
return string.char(y), 3
elseif x == 127 then
@ -46,31 +34,29 @@ function p(x, y)
return string.char(("enart"):byte(x % 5 + 1), ("ndtelh"):byte((x - x % 5) / 5 + 1)), 2
end
function q(w)
t = t .. w
while #t > 1 do
d, a = p(t:byte(), t:byte(2))
b = b .. d
t = t:sub(a)
if #b > 511 then
M(b:sub(1, 512))
b = b:sub(513)
end
end
end
function L(d)
c = c .. d
while #c > 2 do
s = c:byte()
if s < 128 then
s, c = c:sub(1, 1), c:sub(2)
function $engineInput($a0)
$bdBDBuffer = $bdBDBuffer .. $a0
while #$bdBDBuffer > 2 do
$a0 = $bdBDBuffer:byte()
if $a0 < 128 then
$a0 = $bdBDBuffer:sub(1, 1)
$bdBDBuffer = $bdBDBuffer:sub(2)
else
a = c:byte(2) * 256 + c:byte(3) + 1
s, c = w:sub(a, a + s - 125), c:sub(4)
$NTbdBDPtr
$bdBDPtr = $bdBDBuffer:byte(2) * 256 + $bdBDBuffer:byte(3) + 1
$a0 = $bdBDWindow:sub($bdBDPtr, $bdBDPtr + $a0 - 125)
$bdBDBuffer = $bdBDBuffer:sub(4)
$DTbdBDPtr
end
$bdPPBuffer = $bdPPBuffer .. $a0
$bdBDWindow = ($bdBDWindow .. $a0):sub(-2^16)
while #$bdPPBuffer > 1 do
$NTbdPPAdv
$a0, $bdPPAdv = $bdPP($bdPPBuffer:byte(), $bdPPBuffer:byte(2))
$bdPPBuffer = $bdPPBuffer:sub($bdPPAdv)
$DTbdPPAdv
$engineOutput($a0)
end
q(s)
w = (w .. s):sub(-2^16)
end
end

View File

@ -7,8 +7,7 @@ cid = (cid or "UNKNOWN"):sub(1, 7)
local u = require("libs.frw")
local tarData = u.read(tarName)
local tarSectors = math.floor(#tarData / 512)
local algImpl = require(alg .. ".compress")
local instSize = 0
local function put(data)
@ -16,52 +15,31 @@ local function put(data)
instSize = instSize + #data
end
put("--" .. cid .. "\n")
put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
-- TAR File --
local tarData = u.read(tarName)
local tarSectors = math.floor(#tarData / 512)
local instCode = "K=" .. tarSectors .. "\n" .. u.read(alg .. "/instdeco.lua") .. u.read("instbase.lua")
instCode = require("libs.lexcrunch")(instCode)
put(instCode)
-- Installer Lexcrunch Context --
local lexCrunch = require("libs.lexcrunch")()
-- the \x00 is the indicator to start reading
put("--[[\x00")
local installerCore = lexCrunch(u.read("instcore.lua"), {["$$SECTORS"] = tostring(tarSectors)})
local installerHead = lexCrunch(u.read("insthead.lua"), {["$$CORESIZE"] = tostring(#installerCore)})
local installerTail = lexCrunch(u.read("insttail.lua"), {})
-- Installer Compression --
local rawData = installerCore .. tarData
io.stderr:write("compressing...\n")
local compressedData = require(alg .. ".compress")(tarData)
u.write(alg .. "/output.bin", compressedData)
io.stderr:write("compression with " .. alg .. ": " .. #tarData .. " -> " .. #compressedData .. "\n")
-- Program the read-in state machine
local compressionEngine, compressedData = algImpl(rawData, lexCrunch)
-- RISM [[
compressedData = compressedData:gsub("\xFE", "\xFE\xFE")
compressedData = compressedData:gsub("]]", "]\xFE]")
compressedData = "\x00" .. compressedData
-- ]]
io.stderr:write("compression with " .. alg .. ": " .. #rawData .. " -> " .. #compressedData .. "\n")
put(compressedData)
put("]]")
local status = ""
local statusDetail = ""
local blinkI = ""
if instSize > 65536 then
blinkI = "5;31;"
status = " DO NOT SHIP "
statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
elseif instSize > 64000 then
blinkI = "33;"
status = " Shippable * "
statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
else
blinkI = "32;"
status = " All Green "
statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
end
io.stderr:write("\n")
local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
io.stderr:write(ctS .. " " .. ctM .. " " .. ctE)
io.stderr:write(ctS .. status .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
io.stderr:write(ctS .. " " .. ctM .. " " .. ctE)
io.stderr:write("\n")
io.stderr:write(statusDetail .. "\n")
io.stderr:write("\n")
io.stderr:write("Size: " .. instSize .. "\n")
io.stderr:write(" max. 65536\n")
io.stderr:write("\n")
-- Installer Final Generation --
put("--" .. cid .. "\n")
put("--This is released into the public domain. No warranty is provided, implied or otherwise.\n")
put(lexCrunch(installerHead .. compressionEngine .. installerTail, {}))
put("--[[" .. compressedData .. "]]")

View File

@ -1,139 +0,0 @@
-- KOSNEO installer base
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
-- DECOMPRESSION ENGINE PRECEDES THIS CODE --
-- NOTE: upper-case is reserved for this file,
-- lower-case is reserved for the decompression engine
-- A: temporary
-- B: computer
-- C: component
-- D: additional temporary
-- E: read-in state machine
-- F: current file: filename
-- H: remaining bytes to copy/skip
-- I: current file: handle (nil if not writing)
-- J: sectors handled
-- K: sector count (injected during build)
-- L: compression engine data function (set by CE)
-- M: sector handler function (called by CE)
-- O: current character for read-in state machine
-- P: file handle for selfread
-- Q: octal decoding function
-- X: screen address
-- Y: component: gpu
-- Z: component: filesystem
B = computer
C = component
assert(C, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")
X = C.list("screen", true)()
Y = C.list("gpu", true)()
Z = C.proxy(B.getBootAddress())
Z.remove("init.neoi.lua")
Z.rename("init.lua","init.neoi.lua")
P = Z.open("init.neoi.lua","rb")
F = "Starting..."
H = 0
if X and Y then
Y = C.proxy(Y)
Y.bind(X)
Y.setResolution(50, 5)
Y.setBackground(2^24-1)
Y.setForeground(0)
Y.fill(1, 1, 50, 5, "")
Y.fill(1, 2, 50, 1, " ")
Y.set(2, 2, "KittenOS NEO Installer")
end
function Q(A)
if A == "" then return 0 end
return Q(A:sub(1, -2)) * 8 + (A:byte(#A) - 48)
end
J = 0
function M(n)
if H > 0 then
A = math.min(512, H)
H = H - A
if I then
Z.write(I, n:sub(1, A))
if H <= 0 then
Z.close(I)
I = nil
end
end
else
F = n:sub(1, 100):gsub("\x00", "")
-- this sets up the reading/skipping of data
H = Q(n:sub(125, 135))
if F:sub(1, 2) == "./" and F ~= "./" then
F = F:sub(3)
if F:sub(#F) == "/" then
Z.makeDirectory(F)
else
I = Z.open(F, "wb")
if H == 0 then
Z.close(I)
I = nil
end
end
end
end
-- UPDATE DISPLAY --
J = J + 1
if X and Y then
Y.fill(1, 2, 50, 1, " ")
Y.set(2, 2, "KittenOS NEO Installer : " .. F)
Y.fill(2, 4, 48, 1, "")
Y.fill(2, 4, math.ceil(48 * J / K), 1, " ")
end
if J % 8 == 0 then
B.pullSignal(0.01)
end
if J == K then
Z.close(P)
Z.remove("init.neoi.lua")
B.shutdown(true)
end
end
while true do
A = Z.read(P, 64)
D = ""
for i = 1, #A do
-- Read-in state machine
O = A:sub(i, i)
if not E then
if O == "\x00" then
E = 0
end
elseif E == 0 then
if O == "\xFE" then
E = 1
else
D = D .. O
end
else
D = D .. O
E = 0
end
end
L(D)
end
-- COMPRESSED DATA FOLLOWS THIS CODE --

76
inst/instcore.lua Normal file
View File

@ -0,0 +1,76 @@
-- KOSNEO installer base
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
$icScreen = $component.list("screen", true)()
$icGPU = $component.list("gpu", true)()
$icFilename = "Starting..."
$icBytesRemaining = 0
if $icScreen and $icGPU then
$icGPU = $component.proxy($icGPU)
$icGPU.bind($icScreen)
$icGPU.setResolution(50, 5)
$icGPU.setBackground(2^24-1)
$icGPU.setForeground(0)
$icGPU.fill(1, 1, 50, 5, "")
$icGPU.fill(1, 2, 50, 1, " ")
$icGPU.set(2, 2, "KittenOS NEO Installer")
end
function $icOctalToNumber($a0)
if $a0 == "" then return 0 end
return $icOctalToNumber($a0:sub(1, -2)) * 8 + ($a0:byte(#$a0) - 48)
end
$icSectorsRead = 0
$iBlockingLen = 512
function $iBlockingHook($a0)
if $icBytesRemaining > 0 then
$NTicByteAdv
$icByteAdv = math.min(512, $icBytesRemaining)
$icBytesRemaining = $icBytesRemaining - $icByteAdv
if $icFile then
$filesystem.write($icFile, $a0:sub(1, $icByteAdv))
if $icBytesRemaining <= 0 then
$filesystem.close($icFile)
$icFile = nil
end
end
$DTicByteAdv
else
$icFilename = $a0:sub(1, 100):gsub("\x00", "")
-- this sets up the reading/skipping of data
$icBytesRemaining = $icOctalToNumber($a0:sub(125, 135))
if $icFilename:sub(1, 2) == "./" and $icFilename ~= "./" then
$icFilename = $icFilename:sub(3)
if $icFilename:sub(#$icFilename) == "/" then
$filesystem.makeDirectory($icFilename)
else
$icFile = $filesystem.open($icFilename, "wb")
if $icBytesRemaining == 0 then
$filesystem.close($icFile)
$icFile = nil
end
end
end
end
-- UPDATE DISPLAY --
$icSectorsRead = $icSectorsRead + 1
if $icScreen and $icGPU then
$icGPU.fill(1, 2, 50, 1, " ")
$icGPU.set(2, 2, "KittenOS NEO Installer : " .. $icFilename)
$icGPU.fill(2, 4, 48, 1, "")
$icGPU.fill(2, 4, math.ceil(48 * $icSectorsRead / $$SECTORS), 1, " ")
end
if $icSectorsRead % 16 == 0 then
$computer.pullSignal(0.01)
end
if $icSectorsRead == $$SECTORS then
$filesystem.close($readInFile)
$filesystem.remove("init.neoi.lua")
$computer.shutdown(true)
end
end

34
inst/insthead.lua Normal file
View File

@ -0,0 +1,34 @@
-- KOSNEO installer base
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
$computer = computer
$component = component
assert($component, "KittenOS NEO installer: Copy as init.lua to the target disk, then remove other disks & reboot.")
$filesystem = $component.proxy($computer.getBootAddress())
$filesystem.remove("init.neoi.lua")
$filesystem.rename("init.lua", "init.neoi.lua")
$readInFile = $filesystem.open("init.neoi.lua", "rb")
$iBlockingBuffer = ""
$iBlockingLen = $$CORESIZE
$iBlockingHook = function ($a0)
-- This takes over the iBlockingHook.
assert(load($a0))()
end
$engineOutput = function ($a0)
$iBlockingBuffer = $iBlockingBuffer .. $a0
while #$iBlockingBuffer >= $iBlockingLen do
$NTiBlock
$iBlock = $iBlockingBuffer:sub(1, $iBlockingLen)
$iBlockingBuffer = $iBlockingBuffer:sub($iBlockingLen + 1)
$iBlockingHook($iBlock)
$DTiBlock
end
end
-- DECOMPRESSION ENGINE FOLLOWS THIS CODE --

32
inst/insttail.lua Normal file
View File

@ -0,0 +1,32 @@
-- KOSNEO installer base
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
-- DECOMPRESSION ENGINE PRECEDES THIS CODE --
while true do
$readInBlock = $filesystem.read($readInFile, 1024)
for i = 1, #$readInBlock do
-- Read-in state machine
$NTreadInChar
$readInChar = $readInBlock:sub(i, i)
if not $readInState then
if $readInChar == "\x00" then
$readInState = 0
end
elseif $readInState == 0 then
if $readInChar == "\xFE" then
$readInState = 1
else
$engineInput($readInChar)
end
else
$engineInput($readInChar)
$readInState = 0
end
end
$DTreadInChar
end
-- COMPRESSED DATA FOLLOWS THIS CODE --

View File

@ -72,23 +72,77 @@ local function pass(buffer)
return ob
end
return function (op)
-- comment removal
while true do
local np = op:gsub("%-%-[^\n]*\n", " ")
np = np:gsub("%-%-[^\n]*$", "")
if np == op then
break
end
op = np
-- Context creation --
return function ()
local forwardSymTab = {}
local reverseSymTab = {}
local temporaryPool = {}
local possible = {}
for i = 1, 52 do
possible[i] = ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"):sub(i, i)
end
-- stripping
while true do
local np = pass(op)
if np == op then
return np
local function allocate(reason)
for _, v in pairs(possible) do
if not reverseSymTab[v] then
reverseSymTab[v] = reason
return v
end
end
op = np
end
return op
return function (op, defines)
-- symbol replacement
op = op:gsub("%$[%$a-zA-Z0-9]*", function (str)
if str:sub(2, 2) == "$" then
-- defines
assert(defines[str], "no define " .. str)
return defines[str]
elseif str:sub(2, 3) == "NT" then
-- temporaries +
local id = "$" .. str:sub(4)
assert(not forwardSymTab[id], "var already exists: " .. id)
local val = table.remove(temporaryPool, 1)
if not val then val = allocate("temporary") end
forwardSymTab[id] = val
return ""
elseif str:sub(2, 3) == "DT" then
-- temporaries -
local id = "$" .. str:sub(4)
assert(forwardSymTab[id], "no such var: " .. id)
assert(reverseSymTab[forwardSymTab[id]] == "temporary", "var not allocated as temporary: " .. id)
table.insert(temporaryPool, forwardSymTab[id])
forwardSymTab[id] = nil
return ""
else
-- normal handling
if forwardSymTab[str] then
return forwardSymTab[str]
end
local v = allocate(str)
forwardSymTab[str] = v
return v
end
end)
-- comment removal
while true do
local np = op:gsub("%-%-[^\n]*\n", " ")
np = np:gsub("%-%-[^\n]*$", "")
if np == op then
break
end
op = np
end
-- stripping
while true do
local np = pass(op)
if np == op then
return np
end
op = np
end
return op
end
end

36
inst/status.lua Normal file
View File

@ -0,0 +1,36 @@
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
-- Status Screen --
local target = ...
local u = require("libs.frw")
local instSize = #u.read(target)
local status = ""
local statusDetail = ""
local blinkI = ""
if instSize > 65536 then
blinkI = "5;31;"
status = " DO NOT SHIP "
statusDetail = "The installer is too big to ship safely.\nIt's possible it may crash on Tier 1 systems.\nUpgrade the compression system or remove existing code."
elseif instSize > 64000 then
blinkI = "33;"
status = " Shippable * "
statusDetail = "The installer is getting dangerously large.\nReserve further room for bugfixes."
else
blinkI = "32;"
status = " All Green "
statusDetail = "The installer is well within budget with room for features.\nDevelop as normal."
end
io.stderr:write("\n")
local ctS, ctM, ctE = " \x1b[1;" .. blinkI .. "7m", "\x1b[0;7m", "\x1b[0m\n"
io.stderr:write(ctS .. " " .. ctM .. " " .. ctE)
io.stderr:write(ctS .. status .. ctM .. string.format(" %07i ", 65536 - instSize) .. ctE)
io.stderr:write(ctS .. " " .. ctM .. " " .. ctE)
io.stderr:write("\n")
io.stderr:write(statusDetail .. "\n")
io.stderr:write("\n")
io.stderr:write("Size: " .. instSize .. "\n")
io.stderr:write(" max. 65536\n")
io.stderr:write("\n")

View File

@ -1,7 +1,10 @@
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
return function (b)
return b
-- Example compression engine.
-- Given: data, lexCrunch
-- returns compressionEngine, compressedData
return function (data, lexCrunch)
return lexCrunch(" $engineInput = $engineOutput ", {}), data
end

View File

@ -1,13 +0,0 @@
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
t = ""
function L(d)
if not d then return end
t = t .. d
while #t >= 512 do
M(t:sub(1, 512))
t = t:sub(513)
end
end

View File

@ -1,30 +0,0 @@
-- This is released into the public domain.
-- No warranty is provided, implied or otherwise.
-- Installer Compression Verification Tool --
local alg, tarName = ...
local u = require("libs.frw")
io.stderr:write("verifying... ")
local p = u.progress()
local tarData = u.read(tarName)
local total = ""
function M(t)
assert(#t == 512)
total = total .. t
p(#total / #tarData)
end
dofile(alg .. "/instdeco.lua")
L(u.read(alg .. "/output.bin"))
if total ~= tarData then
io.stderr:write("\n" .. #total .. " : " .. #tarData .. "\n")
u.write(alg .. "/vfyerr.bin", total)
error("VERIFICATION FAILURE : see inst/" .. alg .. "/vfyerr.bin!")
end
io.stderr:write("\nverification success\n")

View File

@ -16,7 +16,7 @@ cd ..
# The Installer Creator
cd inst
lua build.lua $1 ../code.tar `git status --porcelain=2 --branch | grep branch.oid | grep -E -o "[0-9a-f]*$" -` > ../inst.lua
lua verify.lua $1 ../code.tar
lua status.lua ../inst.lua
cd ..
# Common Repository Setup Code