From 13f8190cbf1a8b0c2407b06e3d52bfe695da994a Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 13 Dec 2020 15:41:24 -0500 Subject: [PATCH] lzss --- .build/00_setup.lua | 3 + .build/01_velx.lua | 222 ++++++++++++++++++++++++++++++ .build/02_lzss.lua | 92 +++++++++++++ .build/10_bios.lua | 86 ++++++++++++ .build/11_modules.lua | 35 +++++ .build/12_libraries.lua | 33 +++++ .build/20_pkg.lua | 3 + .build/30_selfextract.lua | 22 +++ .build/ff_meta.lua | 17 +++ .buildactions/00_setup.lua | 2 + .buildactions/01_velx.lua | 4 + src/zy-neo/builtins/util_velx.lua | 4 +- 12 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 .build/00_setup.lua create mode 100644 .build/01_velx.lua create mode 100644 .build/02_lzss.lua create mode 100644 .build/10_bios.lua create mode 100644 .build/11_modules.lua create mode 100644 .build/12_libraries.lua create mode 100644 .build/20_pkg.lua create mode 100644 .build/30_selfextract.lua create mode 100644 .build/ff_meta.lua diff --git a/.build/00_setup.lua b/.build/00_setup.lua new file mode 100644 index 0000000..eb9fc93 --- /dev/null +++ b/.build/00_setup.lua @@ -0,0 +1,3 @@ +task("dirs", function() + os.execute("mkdir -p release") +end) \ No newline at end of file diff --git a/.build/01_velx.lua b/.build/01_velx.lua new file mode 100644 index 0000000..2da2d9c --- /dev/null +++ b/.build/01_velx.lua @@ -0,0 +1,222 @@ +-- VELX builder + +--[[---------------------------------------------------------------------------- + LZSS - encoder / decoder + This is free and unencumbered software released into the public domain. + Anyone is free to copy, modify, publish, use, compile, sell, or + distribute this software, either in source code form or as a compiled + binary, for any purpose, commercial or non-commercial, and by any + means. + In jurisdictions that recognize copyright laws, the author or authors + of this software dedicate any and all copyright interest in the + software to the public domain. We make this dedication for the benefit + of the public at large and to the detriment of our heirs and + successors. We intend this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights to this + software under copyright law. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + For more information, please refer to +--]]---------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +local M = {} +local string, table = string, table + +-------------------------------------------------------------------------------- +local POS_BITS = 12 +local LEN_BITS = 16 - POS_BITS +local POS_SIZE = 1 << POS_BITS +local LEN_SIZE = 1 << LEN_BITS +local LEN_MIN = 3 + +-------------------------------------------------------------------------------- +function M.compress(input) + local offset, output = 1, {} + local window = '' + + local function search() + for i = LEN_SIZE + LEN_MIN - 1, LEN_MIN, -1 do + local str = string.sub(input, offset, offset + i - 1) + local pos = string.find(window, str, 1, true) + if pos then + return pos, str + end + end + end + + while offset <= #input do + local flags, buffer = 0, {} + + for i = 0, 7 do + if offset <= #input then + local pos, str = search() + if pos and #str >= LEN_MIN then + local tmp = ((pos - 1) << LEN_BITS) | (#str - LEN_MIN) + buffer[#buffer + 1] = string.pack('>I2', tmp) + else + flags = flags | (1 << i) + str = string.sub(input, offset, offset) + buffer[#buffer + 1] = str + end + window = string.sub(window .. str, -POS_SIZE) + offset = offset + #str + else + break + end + end + + if #buffer > 0 then + output[#output + 1] = string.char(flags) + output[#output + 1] = table.concat(buffer) + end + end + + return table.concat(output) +end + +-------------------------------------------------------------------------------- +function M.decompress(input) + local offset, output = 1, {} + local window = '' + + while offset <= #input do + local flags = string.byte(input, offset) + offset = offset + 1 + + for i = 1, 8 do + local str = nil + if (flags & 1) ~= 0 then + if offset <= #input then + str = string.sub(input, offset, offset) + offset = offset + 1 + end + else + if offset + 1 <= #input then + local tmp = string.unpack('>I2', input, offset) + offset = offset + 2 + local pos = (tmp >> LEN_BITS) + 1 + local len = (tmp & (LEN_SIZE - 1)) + LEN_MIN + str = string.sub(window, pos, pos + len - 1) + end + end + flags = flags >> 1 + if str then + output[#output + 1] = str + window = string.sub(window .. str, -POS_SIZE) + end + end + end + + return table.concat(output) +end + + +local function struct(tbl) + local pat = tbl.endian or "=" + local args = {} + for i=1, #tbl do + local a, b = pairs(tbl[i]) + local k, v = a(b) + args[i] = k + pat = pat .. v + end + return setmetatable({}, {__call=function(_, arg) + --checkArg(1, arg, "string", "table") + if (type(arg) == "string") then + local sval = {string.unpack(pat, arg)} + local rtn = {} + for i=1, #args do + rtn[args[i]] = sval[i] + end + return rtn, sval[#sval] + elseif (type(arg) == "table") then + local sval = {} + for i=1, #args do + sval[i] = arg[args[i]] + end + return string.pack(pat, table.unpack(sval)) + end + end, __len=function() + return string.packsize(pat) + end}) +end + + +local velx_spec = struct { + endian = "<", + {magic="c5"}, + {fver="B"}, + {compression="B"}, + {lver="B"}, + {os="B"}, + {arctype="c4"}, + {psize="I3"}, + {lsize="I3"}, + {ssize="I3"}, + {rsize="I4"} +} + +local function velx_multistep(path, arcpath, args, steps) + local shell_args = "" + for k, v in pairs(args) do + if (k == "PWD") then + shell_args = "cd "..v.."; " .. shell_args + else + shell_args = shell_args .. k .. "=\"".. v .."\" " + end + end + steps.precomp() + local h = io.popen(shell_args.."luacomp "..path.." 2>/dev/null", "r") + local prog = h:read("*a") + h:close() + prog = steps.postcomp(prog) + steps.prearc() + local arc = "" + if arcpath then + h = io.popen("cd "..arcpath.."; find $".."(ls) -depth | tsar -o", "r") + arc = h:read("*a") + h:close() + steps.postarc() + end + if (not args.noz) then + steps.prez() + prog = M.compress(prog) + steps.postz() + end + local header = velx_spec({ + magic = "\27VelX", + compression = (args.noz and 0) or 1, + lver = 0x53, + fver = 1, + os = 0xDA, --da, comrade + psize = #prog, + lsize=0, + ssize=0, + rsize=#arc, + arctype=(arcpath and "tsar") or "" + }) + return header .. prog .. arc +end + +local function velx(path, arcpath, args) + return velx_multistep(path, arcpath, args, { + precomp = function()end, + postcomp = function(prog) return prog end, + prearc = function()end, + postarc = function()end, + prez = function()end, + postz = function()end, + }) +end + +local function velxv2(sections) + +end + +EXPORT.velx = velx +EXPORT.velx_multistep = velx_multistep \ No newline at end of file diff --git a/.build/02_lzss.lua b/.build/02_lzss.lua new file mode 100644 index 0000000..c1ebf76 --- /dev/null +++ b/.build/02_lzss.lua @@ -0,0 +1,92 @@ +local M = {} +local string, table = string, table + +-------------------------------------------------------------------------------- +local POS_BITS = 12 +local LEN_BITS = 16 - POS_BITS +local POS_SIZE = 1 << POS_BITS +local LEN_SIZE = 1 << LEN_BITS +local LEN_MIN = 3 + +-------------------------------------------------------------------------------- +function M.compress(input) + local offset, output = 1, {} + local window = '' + + local function search() + for i = LEN_SIZE + LEN_MIN - 1, LEN_MIN, -1 do + local str = string.sub(input, offset, offset + i - 1) + local pos = string.find(window, str, 1, true) + if pos then + return pos, str + end + end + end + + while offset <= #input do + local flags, buffer = 0, {} + + for i = 0, 7 do + if offset <= #input then + local pos, str = search() + if pos and #str >= LEN_MIN then + local tmp = ((pos - 1) << LEN_BITS) | (#str - LEN_MIN) + buffer[#buffer + 1] = string.pack('>I2', tmp) + else + flags = flags | (1 << i) + str = string.sub(input, offset, offset) + buffer[#buffer + 1] = str + end + window = string.sub(window .. str, -POS_SIZE) + offset = offset + #str + else + break + end + end + + if #buffer > 0 then + output[#output + 1] = string.char(flags) + output[#output + 1] = table.concat(buffer) + end + end + + return table.concat(output) +end + +-------------------------------------------------------------------------------- +function M.decompress(input) + local offset, output = 1, {} + local window = '' + + while offset <= #input do + local flags = string.byte(input, offset) + offset = offset + 1 + + for i = 1, 8 do + local str = nil + if (flags & 1) ~= 0 then + if offset <= #input then + str = string.sub(input, offset, offset) + offset = offset + 1 + end + else + if offset + 1 <= #input then + local tmp = string.unpack('>I2', input, offset) + offset = offset + 2 + local pos = (tmp >> LEN_BITS) + 1 + local len = (tmp & (LEN_SIZE - 1)) + LEN_MIN + str = string.sub(window, pos, pos + len - 1) + end + end + flags = flags >> 1 + if str then + output[#output + 1] = str + window = string.sub(window .. str, -POS_SIZE) + end + end + end + + return table.concat(output) +end + +EXPORT.lzss = M \ No newline at end of file diff --git a/.build/10_bios.lua b/.build/10_bios.lua new file mode 100644 index 0000000..614846a --- /dev/null +++ b/.build/10_bios.lua @@ -0,0 +1,86 @@ +local function mkplat(plat) + local h = io.popen("luacomp src/loader.lua".." 2>/dev/null") + local loader = h:read("*a") + h:close() + local h = io.popen("ZY_PLATFORM="..plat.." luacomp src/zy-neo/neoinit.lua 2>/dev/null | lua5.3 utils/makezbios.lua") + local dat = h:read("*a") + h:close() + os.execute("mkdir -p pkg/bios") + local h = io.open("pkg/bios/"..plat..".bios", "wb") + h:write(string.format(loader, dat)) + h:close() + if (os.execute("[[ $".."(stat --printf=%s pkg/bios/"..plat..".bios) > 4096 ]]")) then + io.stderr:write("WARNING: "..plat.." bios is over 4KiB!\n") + end +end + +--[[function actions.managed_bios() + mkplat("managed") +end + +function actions.initramfs_bios() + mkplat("initramfs") +end + +function actions.prom_bios() + mkplat("prom") +end + +function actions.osdi_bios() + mkplat("osdi") +end + +function actions.bootstrap() + --os.execute("luacomp src/zy-neo/zinit.lua | lua5.3 utils/makezbios.lua > pkg/bios/bootstrap.bin") + local h = io.popen("luacomp src/zy-neo/zinit.lua") + local dat = h:read("*a") + h:close() + local h = io.open("pkg/bios/bootstrap.bin", "wb") + h:write(lzss.compress(dat)) + h:close() +end + +function actions.bios() + actions.managed_bios() + actions.initramfs_bios() + actions.prom_bios() + actions.osdi_bios() + actions.bootstrap() +end]] + +local blist = {} + +local function addbios(name) + blist[#blist+1] = name..".bios" + task(name..".bios", function() + status("build", name) + mkplat(name) + end) +end + +addbios("managed") +addbios("initramfs") +addbios("prom") +addbios("osdi") + +task("bootstrap.bin", function() + local h = io.popen("luacomp src/zy-neo/zinit.lua".." 2>/dev/null") + local dat = h:read("*a") + h:close() + local h = io.open("pkg/bios/bootstrap.bin", "wb") + h:write(EXPORT.lzss.compress(dat)) + h:close() +end) + +task("bios", function() + for i=1, #blist do + dep(blist[i]) + end + dep("bootstrap.bin") +end) + +--[[actions[#actions+1] = "managed_bios" +actions[#actions+1] = "initramfs_bios" +actions[#actions+1] = "prom_bios" +actions[#actions+1] = "osdi_bios" +actions[#actions+1] = "bootstrap"]] \ No newline at end of file diff --git a/.build/11_modules.lua b/.build/11_modules.lua new file mode 100644 index 0000000..8c357b1 --- /dev/null +++ b/.build/11_modules.lua @@ -0,0 +1,35 @@ +local function make_module(mod) + os.execute("mkdir -p pkg/mods") + --print("MOD", mod) + local arc = false + if (os.execute("[[ -d mods/"..mod.."/arc ]]")) then + arc = "mods/"..mod.."/arc" + end + local h = io.open("pkg/mods/"..mod..".velx", "w") + h:write(EXPORT.velx("init.lua", arc, { + PWD = os.getenv("PWD").."/mods/"..mod + })) + h:close() +end + +local mods = {} + +local h = io.popen("ls mods", "r") +for line in h:lines() do + --[[actions["mod_"..line] = function() + make_module(line) + end]] + task("mod_"..line, function() + status("build", line) + make_module(line) + end) + mods[#mods+1] = "mod_"..line +end + +task("allmods", function() + for i=1, #mods do + dep(mods[i]) + end +end) + +--actions[#actions+1] = "allmods" \ No newline at end of file diff --git a/.build/12_libraries.lua b/.build/12_libraries.lua new file mode 100644 index 0000000..ee5558f --- /dev/null +++ b/.build/12_libraries.lua @@ -0,0 +1,33 @@ +local function make_library(mod) + os.execute("mkdir -p pkg/lib") + --print("LIB", mod) + local arc = false + if (os.execute("[[ -d lib/"..mod.."/arc ]]")) then + arc = "lib/"..mod.."/arc" + end + local h = io.open("pkg/lib/"..mod..".velx", "w") + h:write(EXPORT.velx("init.lua", arc, { + PWD = os.getenv("PWD").."/lib/"..mod + })) + h:close() +end + +local lib = {} + +local h = io.popen("ls lib", "r") +for line in h:lines() do + --[[actions["mod_"..line] = function() + make_module(line) + end]] + task("lib_"..line, function() + status("build", line) + make_library(line) + end) + lib[#lib+1] = "lib_"..line +end + +task("alllibs", function() + for i=1, #lib do + dep(lib[i]) + end +end) \ No newline at end of file diff --git a/.build/20_pkg.lua b/.build/20_pkg.lua new file mode 100644 index 0000000..4dfe8a8 --- /dev/null +++ b/.build/20_pkg.lua @@ -0,0 +1,3 @@ +task("makepkg", function() + os.execute("cd pkg; find bios lib mods -depth | lua ../utils/make_tsar.lua > ../release/zorya-neo-update.tsar") +end) \ No newline at end of file diff --git a/.build/30_selfextract.lua b/.build/30_selfextract.lua new file mode 100644 index 0000000..9fab280 --- /dev/null +++ b/.build/30_selfextract.lua @@ -0,0 +1,22 @@ +local function makeselfextract(indir, outfile) + local cwd = os.getenv("PWD") + os.execute("cd "..indir.."; find * -depth | lua "..cwd.."/utils/make_tsar.lua | lua "..cwd.."/utils/mkselfextract.lua > "..cwd.."/"..outfile) +end + +task("installer", function() + os.execute("cp utils/ser.lua pkg/init.lua") + os.execute("mkdir -p pkg/installer_dat") + os.execute("cp installer_dat/bios_list.lua pkg/installer_dat") + os.execute("cp installer_dat/package_list.lua pkg/installer_dat") + os.execute("mkdir -p pkg/installer_dat/lang") + local h = io.popen("ls installer_dat/lang | grep lua", "r") + for line in h:lines() do + os.execute("luacomp installer_dat/lang/"..line.." -O pkg/installer_dat/lang/"..line.." 2>/dev/null") + end + h:close() + makeselfextract("pkg", "release/zorya-neo-installer.lua") +end) + +task("utils", function() + makeselfextract("util", "release/zorya-neo-utils-installer.lua") +end) \ No newline at end of file diff --git a/.build/ff_meta.lua b/.build/ff_meta.lua new file mode 100644 index 0000000..ee2c7d6 --- /dev/null +++ b/.build/ff_meta.lua @@ -0,0 +1,17 @@ +task("clean", function() + os.execute("mkdir -p pkg") + os.execute("rm -rf pkg/*") + + os.execute("mkdir -p release") + os.execute("rm -rf release/*") +end) + +task("all", function() + dep("dirs") + dep("bios") + dep("allmods") + dep("alllibs") + dep("makepkg") + dep("installer") + dep("utils") +end) \ No newline at end of file diff --git a/.buildactions/00_setup.lua b/.buildactions/00_setup.lua index 81d5b11..e2bbb3f 100644 --- a/.buildactions/00_setup.lua +++ b/.buildactions/00_setup.lua @@ -1 +1,3 @@ +unpack = unpack or table.unpack + os.execute("mkdir -p release") \ No newline at end of file diff --git a/.buildactions/01_velx.lua b/.buildactions/01_velx.lua index 048a396..97e6bda 100644 --- a/.buildactions/01_velx.lua +++ b/.buildactions/01_velx.lua @@ -212,4 +212,8 @@ local function velx(path, arcpath, args) prez = function()end, postz = function()end, }) +end + +local function velxv2(sections) + end \ No newline at end of file diff --git a/src/zy-neo/builtins/util_velx.lua b/src/zy-neo/builtins/util_velx.lua index a9d7112..e0da470 100644 --- a/src/zy-neo/builtins/util_velx.lua +++ b/src/zy-neo/builtins/util_velx.lua @@ -119,8 +119,8 @@ do if (sec.id == osid and sec.name == name or sec.id == 0) then self.seek(sec.pos-self.seek()) local dat = self.read(sec.size) - if (sec.tags.compression == "lz4") then - dat = lz4_decompress(dat) + if (sec.tags.compression == "lzss") then + dat = lzss_decompress(dat) end return dat end