From 6237871d36f413bb34f7aa98c036783339076ebe Mon Sep 17 00:00:00 2001 From: 20kdc Date: Tue, 12 Jun 2018 21:45:40 +0100 Subject: [PATCH] Fix some CLAW stuff and document CLAW formats in neo-docs --- claw/C2-Format.md | 3 +- claw/code-claw.lua | 2 +- claw/repo-claw.lua | 3 +- code/apps/app-claw.lua | 8 +- code/apps/svc-app-claw-worker.lua | 45 ++++-- repository/docs/repoauthors/neo-docs | 1 + repository/docs/us-clawf | 230 +++++++++++++++++++++++++++ 7 files changed, 268 insertions(+), 24 deletions(-) create mode 100644 repository/docs/us-clawf diff --git a/claw/C2-Format.md b/claw/C2-Format.md index eaed1dc..36cd23d 100644 --- a/claw/C2-Format.md +++ b/claw/C2-Format.md @@ -62,7 +62,7 @@ svc-app-claw-worker does all package consistency & such work. It can only be run from app-claw, and runs app-claw after it's done. -It takes 4 arguments: +It takes 5 arguments: 1. The target filesystem proxy. 2. The target package name. This package is viewed in app-claw after completion. @@ -72,3 +72,4 @@ It takes 4 arguments: Proxy means it's a filesystem, string means it's an internet base. 4. Checked flag +5. primary inet card proxy, if any diff --git a/claw/code-claw.lua b/claw/code-claw.lua index 464ff8d..1c14e61 100644 --- a/claw/code-claw.lua +++ b/claw/code-claw.lua @@ -165,7 +165,7 @@ return { }, ["app-claw"] = { desc = "KittenOS NEO Package Manager", - v = 3, + v = 5, deps = { "neo" }, diff --git a/claw/repo-claw.lua b/claw/repo-claw.lua index 81c0278..0f5499e 100644 --- a/claw/repo-claw.lua +++ b/claw/repo-claw.lua @@ -24,7 +24,7 @@ return { }, ["neo-docs"] = { desc = "KittenOS NEO system documentation", - v = 4, + v = 5, deps = { "zzz-license-pd" }, @@ -42,6 +42,7 @@ return { "docs/us-nxapp", "docs/us-setti", "docs/us-evrst", + "docs/us-clawf", "docs/ul-seria", "docs/ul-fwrap", "docs/ul-event", diff --git a/code/apps/app-claw.lua b/code/apps/app-claw.lua index 1bf732d..4ec27eb 100644 --- a/code/apps/app-claw.lua +++ b/code/apps/app-claw.lua @@ -262,7 +262,7 @@ local function packageGetBB(src, lclI, srcI, srcW) function (w) w.close() running = false - neo.executeAsync("svc-app-claw-worker", sources[src], packageId, nil, src == "local") + neo.executeAsync("svc-app-claw-worker", sources[src], packageId, nil, src == "local", primaryINet) end }) end @@ -272,7 +272,7 @@ local function packageGetBB(src, lclI, srcI, srcW) function (w) w.close() running = false - neo.executeAsync("svc-app-claw-worker", sources["local"], packageId, sources[src], true) + neo.executeAsync("svc-app-claw-worker", sources["local"], packageId, sources[src], true, primaryINet) end }) end @@ -282,7 +282,7 @@ local function packageGetBB(src, lclI, srcI, srcW) function (w) w.close() running = false - neo.executeAsync("svc-app-claw-worker", sources[src], packageId, sources["local"], true) + neo.executeAsync("svc-app-claw-worker", sources[src], packageId, sources["local"], true, primaryINet) end }) table.insert(buttons, { @@ -290,7 +290,7 @@ local function packageGetBB(src, lclI, srcI, srcW) function (w) w.close() running = false - neo.executeAsync("svc-app-claw-worker", sources[src], packageId, sources["local"], false) + neo.executeAsync("svc-app-claw-worker", sources[src], packageId, sources["local"], false, primaryINet) end }) end diff --git a/code/apps/svc-app-claw-worker.lua b/code/apps/svc-app-claw-worker.lua index fb6123b..eba7e79 100644 --- a/code/apps/svc-app-claw-worker.lua +++ b/code/apps/svc-app-claw-worker.lua @@ -3,7 +3,7 @@ -- svc-app-claw-worker: Who stays stays. Who goes goes. -local callerPkg, _, destProx, packageId, downloadSrc, checked = ... +local callerPkg, _, destProx, packageId, downloadSrc, checked, primaryINet = ... if callerPkg ~= "app-claw" then error("Internal process for app-claw's bidding.") end -- This 'mutex' remains active as long as the process does. neo.requireAccess("r.svc.app-claw-worker", "CLAW mutex") @@ -66,25 +66,36 @@ local opInstall, opRemove function opInstall(packageId, checked) local gback = {} -- the ultimate strategy + local gdir = {} + local preinstall = {} download("data/app-claw/" .. packageId .. ".c2x", wrapLines(function (l) if l:sub(1, 1) == "?" and checked then - if not destProx.exists("data/app-claw/" .. l:sub(2) .. ".c2x") then - opInstall(l:sub(2), true) - end + preinstall[l:sub(2)] = true elseif l:sub(1, 1) == "+" then table.insert(gback, l:sub(2)) elseif l:sub(1, 1) == "/" then - destProx.makeDirectory(l) - assert(destProx.isDirectory(l), "unable to create dir " .. l) + table.insert(gdir, l:sub(2)) end end), downloadSrc) + for _, v in ipairs(gdir) do + destProx.makeDirectory(v) + assert(destProx.isDirectory(v), "unable to create dir " .. v) + end + gdir = nil + for k, _ in pairs(preinstall) do + if not destProx.exists("data/app-claw/" .. k .. ".c2x") then + opInstall(k, true) + end + end + preinstall = nil for _, v in ipairs(gback) do local f = destProx.open(v .. ".C2T", "wb") assert(f, "unable to create download file") - local ok, err = pcall(download, v, function (b) + xpcall(function () + destProx.close(f) + end, download, v, function (b) assert(destProx.write(f, b or ""), "unable to save data") end, downloadSrc) - assert(ok, err) destProx.close(f) end -- CRITICAL SECTION -- @@ -130,15 +141,15 @@ function opRemove(packageId, checked) end end -local ok, err +local function ender(...) + destProx = nil + downloadSrc = nil + primaryINet = nil + neo.executeAsync("app-claw", packageId) + return ... +end if downloadSrc then - ok, err = pcall(opInstall, packageId, checked) + xpcall(ender, opInstall, packageId, checked) else - ok, err = pcall(opRemove, packageId, checked) -end -fsProxy = nil -downloadSrc = nil -neo.executeAsync("app-claw", packageId) -if not ok then - error(err) + xpcall(ender, opRemove, packageId, checked) end diff --git a/repository/docs/repoauthors/neo-docs b/repository/docs/repoauthors/neo-docs index bb9b1f5..2af57e5 100644 --- a/repository/docs/repoauthors/neo-docs +++ b/repository/docs/repoauthors/neo-docs @@ -15,4 +15,5 @@ repository/docs/us-evrst: 20kdc, Public Domain repository/docs/us-nxapp: 20kdc, Public Domain repository/docs/us-perms: 20kdc, Public Domain repository/docs/us-setti: 20kdc, Public Domain +repository/docs/us-clawf: 20kdc, Public Domain diff --git a/repository/docs/us-clawf b/repository/docs/us-clawf new file mode 100644 index 0000000..85bfd70 --- /dev/null +++ b/repository/docs/us-clawf @@ -0,0 +1,230 @@ +As of KittenOS NEO r5, CLAW has been + rewritten to be lighter on memory + usage. This has caused a change in + the formats CLAW uses, and this + has made them non-trivial. + +Note that this documentation should + not be considered the final format + CLAW will ever use. + +There was a format before this, and + a format before that, and there may + be a format after this, and a format + further out into the future. + +So all the CLAW formats over time + will be described here. + +-- CLAW "local.lua" Lua format + +A CLAW "local.lua" file, like the + "-claw.lua" files used in the 'claw' + directory of the OC-KittenOS Git + repository, is a full index of what + the other formats are meant to + represent. + +It is a serialized Lua table, with + the keys being package IDs, + and the values being tables of the + form as follows: + + desc = A description + v = The version number + deps = ipairsable table, list of + packages required for installation + dirs = ipairsable table, list of + the dirs, no pre/post '/'. + files = ipairsable table, list of + the files, no prefixed '/'. + + All mentioned directories must be + in dirs. + +-- CLAWr2 "local.lua" binary format + +The first attempt at saving memory + lead to local.lua being stored in + memory in a binary format. + +This did lead to a reduction in use, + but I found it insufficient. + +The following two notes are from + testing KittenOS NEO r4. + +Even a slightly loaded system with a + simple background service could run + out of memory on the package screen. + +Indeed, even a baseline "just claw + and allmem" reading failed for tests + due to the launcher refusing to work + with CLAW loaded. + +The format is slightly weird, since + it being on disk resulted from a bug + in what was meant to be only an + *in-memory* compression scheme. + +Firstly, I should define a length- + byte-prefixed-string. + +It's a byte, followed by that many + bytes for the contents. + +Secondly, I should define a length- + byte-prefixed list. + +It's a byte, followed by that many + length-prefixed-strings. + +It is a list of entries made up of + the following form: + + LPStr packageId + LPStr desc + u16be v + LPList dirs + LPList files + LPList deps + +The idea here is simply that the Lua + table format is extremely memory- + inefficient, prioritizing speed, so + by packing stuff into a string, it + can save some memory. + +However, more code was needed to + encode and decode this format, so + the savings weren't great. + +-- CLAWr5 (C2) ".c2x"/".c2p"/".c2l" + +This is the current format, and is + hopefully the last format that will + be needed. + +The solution was to rewrite CLAW, to + make the GUI stay a GUI and separate + the logic up via separate file + formats entirely. + +All 3 of these formats are line-based + formats that use "\n" to split their + entries, and they are rather strict + about this. + +All 3 kinds of files are found in the + data/app-claw/ directory, and their + names matter. + +--- C2L + +".c2l" is the simplest format, simply + being a file listing of the app-claw + directory. It is only used for a + remote repository, where it is + called "local.c2l". + +--- C2P + +".c2p" is the second simplest. +It is the raw text, unlinewrapped, of + the package description panel. + +The name given to it is important: + "PACKAGE.VERSION.c2p" + +Thus, the app-claw that currently + exists has a c2p file with name: + "app-claw.5.c2p" + +This is only used by the app-claw UI, + for version numbers, descriptions, + and to show the entries in the first + place. Since none of these matter to + the installation/removal process, + they are ignored completely by the + installer/remover. + +--- C2X + +The app-claw UI runs a separate + program to perform actions based on + the .c2p files it finds and what the + user chooses to do. + +These actions summarize to 4 kinds of + possible action between any readable + source (for installations) and a + local filesystem for a given package + ID: + +1. Install with dependencies +2. Install without dependencies +3. Remove checking for dep. errors +4. Remove regardless of dep. errors + +C2X files use the name "PACKAGE.c2x", + and following the example previously + the app-claw file is "app-claw.c2x". + +C2X files solely serve the role of + dependency, install, and removal + instructions, and thus do not have + any user-friendly information such + as a description. + +The first character of each line in + the file describes what the line + does. + +'+' means the remainder is a filename + to be installed/removed. + CLAW will fail to overwrite a file + that already exists. + Upgrades are handled by an unchecked + package removal prior to install. + +'/' means the remainder's a directory + to be added on installation. + +'?' means the remainder's a package + that this package requires. + +Note that the C2P and C2X files are + NOT automatically installed/removed, + the C2X file must specify them + explicitly. Otherwise, a package may + enter some errant states: + +1. Not installed according to UI, but + installed for dependency purposes, + and could be theoretically removed + (no C2P, but a C2X) + +2. Installed according to UI, but + cannot be removed, cannot be + reinstalled, and does not count + for dependency purposes + (no C2X, but a C2P) + +3. Completely uninstalled according + to both UI and dependencies, and + cannot be removed or reinstalled + (no C2P or C2X) + +C2X files are entirely responsible + for the management of packages, and + a basic CLAW client could be made + with only support for C2L and C2X, + C2P files being handled as part of + C2X handling but having no inherent + meaning of their own. + +This client would not have versioning + information or other such things, + but it would work. +