From d7465884d69aa0ee749cd694477ed07002930572 Mon Sep 17 00:00:00 2001 From: Cookie Engineer Date: Tue, 10 Sep 2024 19:49:57 +0200 Subject: [PATCH] :balloon: Epoche 2 / Golang Refactor --- PKGBUILD | 15 +- README.md | 281 +------- TODO.md | 8 + build.sh | 51 +- install.sh | 17 - pacman-backup.js | 1042 --------------------------- source/actions/Serve.go | 255 ++++++- source/cmds/pacman-backup/main.go | 17 + source/pacman/Download.go | 3 +- source/pacman/IsDatabaseFilename.go | 2 + 10 files changed, 342 insertions(+), 1349 deletions(-) create mode 100644 TODO.md delete mode 100644 install.sh delete mode 100755 pacman-backup.js diff --git a/PKGBUILD b/PKGBUILD index 1e4f256..4ea495c 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,12 +1,11 @@ # Maintainer: Cookie Engineer <@cookiengineer> pkgname=pacman-backup -pkgver=r7.e8dd759 +pkgver=r16.7a7623c pkgrel=1 -pkgdesc='Backup tool for off-the-grid updates via portable USB sticks or (mesh) LAN networks.' +pkgdesc='Pacman Backup tool for off-the-grid updates via portable USB sticks or (mesh) LAN networks.' arch=('i686' 'x86_64' 'armv6h' 'armv7h' 'aarch64') -makedepends=('git') -depends=('nodejs') +makedepends=('git' 'go') url='https://github.com/cookiengineer/pacman-backup' license=('GPL') provides=('pacman-backup') @@ -20,8 +19,12 @@ pkgver() { printf "r%s.%s" "$(git rev-list --count HEAD)" "$(git rev-parse --short HEAD)" } +build() { + cd "${srcdir}/${_gitname}"; + bash build.sh; +} package() { - cd "${srcdir}/${_gitname}" - install -Dm755 pacman-backup.js "$pkgdir/usr/bin/pacman-backup"; + cd "${srcdir}/${_gitname}"; + install -Dm755 "build/pacman-backup_linux_amd64" "$pkgdir/usr/bin/pacman-backup"; } diff --git a/README.md b/README.md index f1085e9..10598b8 100644 --- a/README.md +++ b/README.md @@ -1,299 +1,82 @@ # Pacman Backup -Backup tool for off-the-grid updates via portable USB sticks or (mesh) LAN networks. +Offline Pacman Cache management tool that allows off-the-grid updates via +sneakernet (usb drives) or mesh Wi-Fi networks. ## Share Updates via USB Drive -In the below example, make sure that `pacman-usbdrive` is writable. -Replace the path with the correct one that points to your mounted USB drive. +In the below example, make sure that `usb-drive` is writable. Replace the path with +the correct one that points to your mounted USB drive. -**Step 1**: - -On the machine with internet connection, insert and mount the USB drive. - -Use `pacman-backup archive` to copy the package cache to the backup folder. -Use `pacman-backup cleanup` to remain only the latest version of each package. - -The `archive` action will also copy the necessary database files for `pacman -Sy`. +**Step 1**: On the machine _with_ internet connection, insert and mount the USB drive. ```bash # Machine with internet connection - -sudo pacman -Sy; -sudo pacman -Suw; - -pacman-backup archive /run/media/$USER/pacman-usbdrive; -pacman-backup cleanup /run/media/$USER/pacman-usbdrive; +pacman-backup download /run/media/$USER/usb-drive; +pacman-backup cleanup /run/media/$USER/usb-drive; sync; -# Unmount the USB drive and walk to other machine +# Unmount the USB drive and sneak/walk to other machine ``` -**Step 2**: - -On the machine without internet connection, insert and mount the USB drive. - -Copy the necessary database files to pacman's `sync` folder (which is `$DBPath/sync` of `pacman.conf`). - -Use `pacman-backup upgrade` to update from the backupfolder. -This will output the pacman command that you should verify manually before executing it. +**Step 2**: On the machine _without_ internet connection, insert and mount the USB drive. ```bash # Machine without internet connection - -sudo cp /run/media/$USER/pacman-usbdrive/sync/*.db /var/lib/pacman/sync/; -pacman-backup upgrade /run/media/$USER/pacman-usbdrive; - -# :: Use this to install upgrades from cache: -# (...) pacman -U command (...) +pacman-backup upgrade /run/media/$USER/usb-drive; ``` +## Share Updates via Wi-Fi Mesh Network -## Share Updates via LAN Connection - -In the below example, the machine with internet connection has the IP `192.168.0.10`. +In the below example, the machine _with_ internet connection has the IP `192.168.0.10`. Replace the IP with the correct one that matches your setup. If in doubt, use `ip` or `ifconfig`. -**Step 1**: - -On the machine with internet connection, connect the LAN cable (and internet connection). - -Use `pacman-backup serve` to start a pacman server that can be used by other pacman clients. +**Step 1**: On the machine with internet connection, download all updates and serve them as +local pacman archive mirror. ```bash # Machine with internet connection - -sudo pacman -Sy; -sudo pacman -Suw; - +sudo pacman-backup download; pacman-backup serve; ``` -**Step 2**: - -On the machine without internet connection, connect the LAN cable and verify that the server -(the machine with internet connection) is reachable via its IP. If in doubt, use `ping`. - -Use `pacman-backup download 192.168.0.10` to download the packages to pacman's package cache. - -Use `pacman-backup upgrade` to update from the local pacman cache. -This will output the pacman command that you should verify manually before executing it. +**Step 2**: On the machine _without_ direct internet connection, download updates from the +local pacman archive mirror. ```bash # Machine without internet connection - -sudo pacman-backup download 192.168.0.10; -pacman-backup upgrade; +sudo pacman-backup download http://192.168.0.10:15678/; +sudo pacman-backup upgrade; ``` -## Share Updates via LAN Cache Proxy - -`pacman-backup` can also emulate a local Cache Proxy for other pacman clients. -If `pacman-backup serve` is running on the machine with internet connection, it -can be used for `pacman` directly. - -Note that if the packages don't exist, they will appear in the logs but aren't downloaded -directly; and that partial upgrades are not officially supported by Arch Linux. -In the below example, the machine with internet connection has the IP `192.168.0.10`. -Replace the IP with the correct one that matches your setup. If in doubt, use `ip` or `ifconfig`. - -**Step 1**: - -On the machine with internet connection, connect the LAN cable (and internet connection). - -Use `pacman-backup serve` to start a pacman server that can be used by other pacman clients. +## Manual Export and Import of Database Files and Package Cache +If you don't trust automated upgrades and want to use `pacman` directly, that's fine. You +can do so by using `export` on the machine with internet connection and `import` on the +machine without internet connection. ```bash # Machine with internet connection +sudo pacman -Syuw; +pacman-backup export /run/media/$USER/usb-drive; +sync; -sudo pacman -Sy; -sudo pacman -Suw; - -pacman-backup serve; -``` - -**Step 2**: - -Modify the `/etc/pacman.d/mirrorlist` to have as a first entry the following line: - -```conf -# Machine without internet connection -# Pacman Mirrorlist for local server - -Server = http://192.168.0.10:15678 +# Unmount the USB drive and sneak/walk to other machine ``` -Use `pacman -Sy` and `pacman -Su` to update from the Cache Proxy. - ```bash # Machine without internet connection +sudo pacman-backup import /run/media/$USER/usb-drive; +sync; -sudo pacman -Sy; -sudo pacman -Su; # or use -Suw -``` - - -## Advanced Usage - -### Available CLI parameters - -The `pacman-backup` script tries to identify parameters dynamically, which means that -they do not have to be written in a specific order. However, to avoid confusion, the -parameters are detected in different ways; and that is documented here: - -- `ACTION` can be either of `archive`, `cleanup`, `download`, `serve`, or `upgrade`. Defaulted with `null` (and shows help). -- `FOLDER` has to start with `/`. Defaulted with `null` (and uses `/var/cache/pacman/pkg` and `/var/lib/pacman/sync`). -- `MIRROR` has to start with `https://` and contain the full `mirrorlist` syntaxed URL. Supported variables are `$arch` and `$repo`. Defaulted with `https://arch.eckner.net/os/$arch`. -- `SERVER` has to be an `IPv4` separated by `.` or an `IPv6` separated by `:`. Defaulted with `null`. -- `SERVER` can optionally be a `hostname`. If it is a `hostname`, make sure it's setup in the `/etc/hosts` file. - -Full example (which won't make sense but is parsed correctly): - -```bash -# IMPORTANT: MIRROR string needs to be escaped, otherwise bash will try to replace it. - -pacman-backup download /run/media/cookiengineer/my-usb-drive "https://my.own.mirror/archlinux/\$repo/os/\$arch" 192.168.0.123; - -# :: pacmab-backup download -# -> FOLDER: "/run/media/cookiengineer/my-usb-drive" -# -> MIRROR: "https://my.own/mirror/archlinux/$repo/os/$arch" -# -> SERVER: "192.168.0.123" -# -> USER: "cookiengineer" -``` - -### archive - -`archive` allows to backup everything to a specified folder. It copies the files from -`/var/cache/pacman/pkg` and `/var/lib/pacman/sync` into `$FOLDER/pkgs` and `$FOLDER/sync`. - -```bash -# copy local packages to /target/folder/pkgs -# copy local database to /target/folder/sync - -pacman-backup archive /target/folder; -``` - -### cleanup - -`cleanup` allows to cleanup the package cache in a way that only the latest version of -each package is kept (for each architecture). - -If no folder is specified, it will clean up `/var/cache/pacman/pkg`. -If a folder is specified, it will clean up `$FOLDER/pkgs`. - -```bash -# cleanup /var/cache/pacman/pkg - -sudo pacman-backup cleanup; -``` - -```bash -# cleanup /target/folder/pkgs - -pacman-backup cleanup /target/folder; -``` - -### download - -`download` allows to download packages from a `pacman-backup serve` based server. - -If no folder and a server is specified, it will run `pacman -Sy` in advance and download to `/var/cache/pacman/pkg`. -If a folder and a server is specified, it will run `pacman -Sy` in advance and download to `$FOLDER/pkgs`. - -If no folder and no server is specified, it will generate a URL list of packages -that can be copy/pasted into a download manager of choice (e.g. uGet or jdownloader). - -```bash -# download packages to /var/cache/pacman/pkg -# download database to /var/lib/pacman/sync - -sudo pacman-backup download 1.3.3.7; -``` - -```bash -# download packages to /target/folder/pkgs -# download database to /target/folder/sync - -pacman-backup download 1.3.3.7 /target/folder; -``` - -```bash -# generate HTTP/S URL list for packages that need downloading - -pacman-backup download; -``` - -### serve - -`serve` allows to start a `pacman` server that can be used as a local mirror. - -If no folder is specified, it will serve from `/var/cache/pacman/pkg` and `/var/lib/pacman/sync`. -If a folder is specified, it will serve from `$FOLDER/pkgs` and `$FOLDER/sync`. - -```bash -# serve packages from /var/cache/pacman/pkg -# serve database from /var/lib/pacman/sync - -pacman-backup serve; -``` - -```bash -# serve packages from /source/folder/pkgs -# serve database from /source/folder/sync - -pacman-backup serve /source/folder; -``` - -### upgrade - -`upgrade` allows to generate an executable `pacman` command that uses the specified -cache folder as a package source by leveraging the `--cachedir` parameter. - -If no folder is specified, it will upgrade from `/var/cache/pacman/pkg`. -If a folder is specified, it will upgrade from `$FOLDER/pkgs`. - -```bash -# generate upgrade command via packages from /var/cache/pacman/pkg - -pacman-backup upgrade; -``` - -```bash -# generate upgrade command via packages from /source/folder/pkgs - -pacman-backup upgrade /source/folder; +sudo pacman -Su; ``` -### upgrade (Partial Upgrades) - -`upgrade` also prints out a command for missing packages that need downloading. - -In the scenario that the local database says that more packages need to be downloaded -to update everything, it will output an additional command that is prepared to let -`pacman` download the packages to the specified folder. -This is helpful in the scenario that the "offline machine" has more packages installed -than the "online machine" (so that more packages need to be downloaded to fully -upgrade the "offline machine"). +# License -Then you can simply copy/paste the generated command to a text file and execute it -next time you're online - in order to automatically download everything that's -required for the "offline machine". - -```bash -# Example output for partial upgrade scenario -# (executed on machine without internet connection) - -pacman-backup upgrade 1.3.3.7; - -# :: Use this to install upgrades from cache: -# (...) pacman -U command (...) - -# :: Use this to download upgrades into cache: -# (...) pacman -Sw --cachedir command (...) -``` +GPL3 diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..c4371f6 --- /dev/null +++ b/TODO.md @@ -0,0 +1,8 @@ + +# actions/Cleanup + +- [ ] Implement a cleanup mechanism +- [ ] Keep only the newest version of package files in the cache + +- [ ] Maybe harfbuzz and icu need to be preserved in older versions? + diff --git a/build.sh b/build.sh index 3add7e0..afe3013 100644 --- a/build.sh +++ b/build.sh @@ -3,49 +3,34 @@ GO="$(which go 2> /dev/null)"; ROOT="$(pwd)"; - - build() { - local errors=0; - - IFS=$'\n' read -d "" -ra variants <<< "${1// /$'\n'}"; unset IFS; - - for variant in ${variants[@]}; do - - go_os="${variant%%:*}"; - go_arch="${variant##*:}"; + local go_os="${1}"; + local go_arch="${2}"; - if [[ ! -d "${ROOT}/build/${go_os}" ]]; then - mkdir -p "${ROOT}/build/${go_os}"; - fi; - - if [[ "${go_os}" == "windows" ]]; then - cd "${ROOT}/source"; - env CGO_ENABLED=0 GOOS="${go_os}" GOARCH="${go_arch}" ${GO} build -o "${ROOT}/build/${go_os}/pacman-backup-${go_arch}.exe" "${ROOT}/source/cmds/pacman-backup/main.go"; - else - cd "${ROOT}/source"; - env CGO_ENABLED=0 GOOS="${go_os}" GOARCH="${go_arch}" ${GO} build -o "${ROOT}/build/${go_os}/pacman-backup-${go_arch}" "${ROOT}/source/cmds/pacman-backup/main.go"; - fi; - - if [[ $? == 0 ]]; then - echo -e "- Build ${go_tags}: ${go_os} / ${go_arch} [\e[32mok\e[0m]"; - else - echo -e "- Build ${go_tags}: ${go_os} / ${go_arch} [\e[31mfail\e[0m]"; - errors=$((errors+1)); - fi; + if [[ ! -d "${ROOT}/build" ]]; then + mkdir -p "${ROOT}/build"; + fi; - done; + if [[ "${go_os}" == "windows" ]]; then + cd "${ROOT}/source"; + env CGO_ENABLED=0 GOOS="${go_os}" GOARCH="${go_arch}" ${GO} build -o "${ROOT}/build/pacman-backup-${go_os}_${go_arch}.exe" "${ROOT}/source/cmds/pacman-backup/main.go"; + else + cd "${ROOT}/source"; + env CGO_ENABLED=0 GOOS="${go_os}" GOARCH="${go_arch}" ${GO} build -o "${ROOT}/build/pacman-backup-${go_os}_${go_arch}" "${ROOT}/source/cmds/pacman-backup/main.go"; + fi; - if [[ ${errors} == 0 ]]; then + if [[ $? == 0 ]]; then + echo -e "- Build ${go_os} / ${go_arch}: [\e[32mok\e[0m]"; + return 1; + else + echo -e "- Build ${go_os} / ${go_arch}: [\e[31mfail\e[0m]"; return 0; fi; - return 1; - } -build "linux:amd64"; +build "linux" "amd64"; diff --git a/install.sh b/install.sh deleted file mode 100644 index a4b0924..0000000 --- a/install.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"; - - -if [[ "$(which node 2>/dev/null)" == "" ]]; then - sudo pacman -S --needed --noconfirm nodejs; -fi; - - -if [ ! -x /usr/bin/pacman-backup ]; then - - sudo cp "$DIR/pacman-backup.js" /usr/bin/pacman-backup; - sudo chmod +x /usr/bin/pacman-backup; - -fi; - diff --git a/pacman-backup.js b/pacman-backup.js deleted file mode 100755 index 6fbe215..0000000 --- a/pacman-backup.js +++ /dev/null @@ -1,1042 +0,0 @@ -#!/usr/bin/env node - -const Buffer = require('buffer').Buffer; -const dirname = require('path').dirname; -const fs = require('fs'); -const http = require('http'); -const os = require('os'); -const spawn = require('child_process').spawnSync; - -const ARCH = ((arch) => { - if (arch === 'arm') return 'armv7h'; - if (arch === 'arm64') return 'aarch64'; - if (arch === 'x32') return 'i686'; - if (arch === 'x64') return 'x86_64'; - return 'any'; -})(process.arch); -const ARGS = Array.from(process.argv).slice(2); -const ACTION = /^(archive|cleanup|download|serve|upgrade)$/g.test((ARGS[0] || '')) ? ARGS[0] : null; -const FOLDER = ARGS.find((v) => v.startsWith('/')) || null; -const MIRROR = ARGS.find((v) => v.startsWith('https://')) || 'https://arch.eckner.net/os/$arch'; -const SERVER = ARGS.find((v) => (v.includes(':') || v.includes('.'))) || ARGS.find((v) => v !== ACTION && v !== FOLDER && v !== MIRROR) || null; -const USER = process.env.SUDO_USER || process.env.USER || process.env.USERNAME; - - - -if (ACTION !== null) { - - console.log(':: pacman-backup ' + ACTION); - console.log(' -> FOLDER: ' + (FOLDER !== null ? ('"' + FOLDER + '"') : '(none)')); - console.log(' -> MIRROR: ' + (MIRROR !== null ? ('"' + MIRROR + '"') : '(none)')); - console.log(' -> SERVER: ' + (SERVER !== null ? ('"' + SERVER + '"') : '(none)')); - console.log(' -> USER: ' + (USER !== null ? ('"' + USER + '"') : '(none)')); - console.log(''); - -} - - - -/* - * HELPERS - */ - -const toConfig = (server) => ` -[options] -HoldPkg = pacman glibc -Architecture = auto -IgnorePkg = linux linux-headers linux-firmware -CheckSpace -SigLevel = Required DatabaseOptional -LocalFileSigLevel = Optional - -[core] -Server = http://${server}:15678 - -[extra] -Server = http://${server}:15678 - -[community] -Server = http://${server}:15678 - -[multilib] -Server = http://${server}:15678`; - -const copy_file = (source, target, callback) => { - - let called = false; - let done = (err) => { - if (called === false) { - called = true; - callback(err); - } - }; - - - let read = fs.createReadStream(source); - let write = fs.createWriteStream(target); - - read.on('error', (err) => done(err)); - write.on('error', (err) => done(err)); - write.on('close', () => done(null)); - - read.pipe(write); - -}; - -const diff_version = (a, b) => { - - let chunk_a = ''; - let chunk_b = ''; - let rest = false; - - for (let i = 0, il = Math.max(a.length, b.length); i < il; i++) { - - let ch_a = a[i]; - let ch_b = b[i]; - - if (rest === true || ch_a !== ch_b) { - - if (ch_a !== undefined) chunk_a += ch_a; - if (ch_b !== undefined) chunk_b += ch_b; - - rest = true; - - } - - } - - return [ chunk_a, chunk_b ]; - -}; - -const download = (url, callback) => { - - http.get(url, (response) => { - - let buffers = []; - - response.setEncoding('binary'); - response.on('data', (chunk) => { - buffers.push(Buffer.from(chunk, 'binary')); - }); - response.on('end', () => { - callback(Buffer.concat(buffers)); - }); - - }).on('error', () => { - callback(null); - }); - -}; - -const _mkdir = function(path) { - - let is_directory = false; - - try { - - let stat1 = fs.lstatSync(path); - if (stat1.isSymbolicLink()) { - - let tmp = fs.realpathSync(path); - let stat2 = fs.lstatSync(tmp); - if (stat2.isDirectory()) { - is_directory = true; - } - - } else if (stat1.isDirectory()) { - is_directory = true; - } - - } catch (err) { - - if (err.code === 'ENOENT') { - - if (_mkdir(dirname(path)) === true) { - fs.mkdirSync(path, 0o777 & (~process.umask())); - } - - try { - - let stat2 = fs.lstatSync(path); - if (stat2.isSymbolicLink()) { - - let tmp = fs.realpathSync(path); - let stat3 = fs.lstatSync(tmp); - if (stat3.isDirectory()) { - is_directory = true; - } - - } else if (stat2.isDirectory()) { - is_directory = true; - } - - } catch (err) { - // Ignore - } - - } - - } - - return is_directory; - -}; - -const parse_pkgname = (file) => { - - if (file.endsWith('.pkg.tar.xz')) { - file = file.substr(0, file.length - 11); - } - - if (file.endsWith('.pkg.tar.zst')) { - file = file.substr(0, file.length - 12); - } - - - let arch = file.split('-').pop(); - let tmp = file.split('-').slice(0, -1); - - let name = ''; - let version = ''; - - for (let t = 0, tl = tmp.length; t < tl; t++) { - - let chunk = tmp[t]; - let ch = chunk.charAt(0); - - if ( - /^([A-Za-z0-9_+.]+)$/g.test(chunk) - && /[A-Za-z]/g.test(ch) - ) { - if (name.length > 0) name += '-'; - name += chunk; - } else { - if (version.length > 0) version += '-'; - version += chunk; - } - - } - - - let check = (name + '-' + version) === tmp.join('-'); - if (check === true) { - - return { - name: name, - version: version, - arch: arch - }; - - } else { - - return null; - - } - -}; - -const read_databases = (path, callback) => { - - let cache = []; - - fs.readdir(path, (err, files) => { - - if (!err) { - - files.filter((file) => { - return file.endsWith('.db'); - }).forEach((file) => { - cache.push(file); - }); - - callback(cache); - - } - - }); - -}; - -const read_file = (path, callback) => { - - fs.readFile(path, (err, data) => { - callback(err ? null : data); - }); - -}; - -const read_packages = (path, callback) => { - - let cache = []; - - fs.readdir(path, (err, files) => { - - if (!err) { - - files.filter((file) => { - return file.endsWith('.pkg.tar.xz') || file.endsWith('.pkg.tar.zst'); - }).filter((file) => { - return file.includes('-'); - }).forEach((file) => { - cache.push(file); - }); - - callback(cache); - - } - - }); - -}; - -const read_upgrades = (callback) => { - - let handle = spawn('pacman', [ - '-Sup', - '--print-format', - '%n /// %v /// %r /// %l' - ], { - cwd: '/tmp' - }); - - let stdout = (handle.stdout || '').toString().trim(); - let stderr = (handle.stderr || '').toString().trim(); - let upgrades = []; - - if (stderr.length === 0) { - - let lines = stdout.split('\n').filter((l) => l.startsWith('::') === false && l.includes('///')); - if (lines.length > 0) { - - upgrades = lines.map((line) => { - - let file = line.split(' /// ')[3].split('/').pop(); - let arch = null; - - if (file.endsWith('.pkg.tar.xz')) { - arch = file.substr(0, file.length - 11).split('-').pop(); - } - - if (file.endsWith('.pkg.tar.zst')) { - arch = file.substr(0, file.length - 12).split('-').pop(); - } - - return { - file: file, - name: line.split(' /// ')[0], - version: line.split(' /// ')[1], - arch: arch, - repo: line.split(' /// ')[2] - }; - - }); - - } - - } - - callback(upgrades); - -}; - -const remove_file = (target, callback) => { - - fs.unlink(target, (err) => { - callback(err ? err : null); - }); - -}; - -const serve = (path, res) => { - - read_file(path, (buffer) => { - - let file = path.split('/').pop(); - - if (buffer !== null) { - - console.log(':: served "' + file + '"'); - res.writeHead(200, { - 'Content-Type': 'application/octet-stream', - 'Content-Length': Buffer.byteLength(buffer) - }); - res.write(buffer); - res.end(); - - } else { - - console.log(':! Cannot serve "' + file + '"'); - res.writeHead(404, {}); - res.end(); - - } - - }); - -}; - -const serve_with_range = (path, range, res) => { - - read_file(path, (buffer) => { - - let file = path.split('/').pop(); - - if (buffer !== null) { - - let total = Buffer.byteLength(buffer); - let start = range.start || 0; - let end = range.end || total; - let size = (end - start); - - - console.log(':: served "' + file + '" (' + size + ' bytes)'); - res.writeHead(206, { - 'Accept-Ranges': 'bytes', - 'Content-Type': 'application/octet-stream', - 'Content-Length': size, - 'Content-Range': 'bytes ' + start + '-' + end + '/' + total - }); - res.write(buffer.slice(start, end)); - res.end(); - - } else { - - console.log(':! Cannot serve "' + file + '"'); - res.writeHead(404, {}); - res.end(); - - } - - }); - -}; - -const sortByVersion = function(a, b) { - - let [ diff_a, diff_b ] = diff_version(a, b); - - if (diff_a === '' && diff_b === '') { - return 0; - } - - - // "+5+gf0d77228-1" vs. "-1" - let special_a = diff_a.startsWith('+'); - let special_b = diff_b.startsWith('+'); - - if (special_a === true && special_b === true) { - - return 0; - - } else if (special_a === true && diff_b === '-1') { - - return -1; - - } else if (special_b === true && diff_a === '-1') { - - return 1; - - } else if (special_a === false && special_b === false) { - - let dot_a = diff_a.includes('.'); - let dot_b = diff_b.includes('.'); - let cha_a = /[a-z]/g.test(diff_a); - let cha_b = /[a-z]/g.test(diff_b); - - if (cha_a === true && cha_b === true) { - - let ver_a = diff_a.toLowerCase(); - let ver_b = diff_b.toLowerCase(); - - return ver_a > ver_b ? -1 : 1; - - } else if (dot_a === true && dot_b === true) { - - let tmp_a = diff_a.split('.'); - let tmp_b = diff_b.split('.'); - let tl_a = tmp_a.length; - let tl_b = tmp_b.length; - let t_max = Math.min(tl_a, tl_b); - - for (let t = 0; t < t_max; t++) { - - let ch_a = parseInt(tmp_a[t], 10); - let ch_b = parseInt(tmp_b[t], 10); - - if (!isNaN(ch_a) && !isNaN(ch_b)) { - - if (ch_a > ch_b) { - return -1; - } else if (ch_b > ch_a) { - return 1; - } - - } - - } - - } else if ( - (dot_a === true && dot_b === false) - || (dot_a === false && dot_b === true) - ) { - - let tmp_a = diff_a.split('-')[0]; - let tmp_b = diff_b.split('-')[0]; - - let ver_a = parseInt(tmp_a, 10); - if (tmp_a.includes('.')) { - ver_a = parseFloat(tmp_a, 10); - } - - let ver_b = parseInt(tmp_b, 10); - if (tmp_b.includes('.')) { - ver_b = parseFloat(tmp_b, 10); - } - - return ver_a > ver_b ? -1 : 1; - - } else { - - let ch_a = parseInt(diff_a, 10); - let ch_b = parseInt(diff_b, 10); - - if (ch_a > ch_b) { - return -1; - } else if (ch_b > ch_a) { - return 1; - } - - } - - } - - - return 0; - -}; - -const write_file = (target, buffer, callback) => { - - let encoding = 'binary'; - - if (typeof buffer === 'string') { - encoding = 'utf8'; - } - - fs.writeFile(target, buffer, encoding, (err) => { - - if (!err) { - callback(true); - } else { - callback(false); - } - - }); - -}; - - - -/* - * IMPLEMENTATION - */ - -if (ACTION === 'archive' && FOLDER !== null) { - - _mkdir(FOLDER + '/pkgs'); - _mkdir(FOLDER + '/sync'); - - read_packages(FOLDER + '/pkgs', (cache) => { - - read_packages('/var/cache/pacman/pkg', (packages) => { - - packages.filter((file) => { - return cache.includes(file) === false; - }).forEach((file) => { - - copy_file('/var/cache/pacman/pkg/' + file, FOLDER + '/pkgs/' + file, (err) => { - if (!err) console.log(':: archived pkgs/' + file); - }); - - }); - - }); - - }); - - read_databases('/var/lib/pacman/sync', (databases) => { - - databases.filter((file) => { - return file !== 'testing.db'; - }).forEach((file) => { - - copy_file('/var/lib/pacman/sync/' + file, FOLDER + '/sync/' + file, (err) => { - if (!err) console.log(':: archived sync/' + file); - }); - - }); - - }); - -} else if (ACTION === 'cleanup') { - - let pkgs_folder = '/var/cache/pacman/pkg'; - - if (FOLDER !== null) { - pkgs_folder = FOLDER + '/pkgs'; - _mkdir(FOLDER + '/pkgs'); - } - - - read_packages(pkgs_folder, (cache) => { - - let database = { - 'any': {}, - 'armv7h': {}, - 'aarch64': {}, - 'i686': {}, - 'x86_64': {}, - }; - - cache.forEach((file) => { - - let pkg = parse_pkgname(file); - if (pkg === null) { - - console.log(':! Cannot recognize version scheme of ' + file); - - } else { - - let entry = database[pkg.arch][pkg.name] || null; - if (entry === null) { - - database[pkg.arch][pkg.name] = { - arch: pkg.arch, - name: pkg.name, - versions: [ pkg.version ] - }; - - } else { - entry.versions.push(pkg.version); - } - - } - - }); - - for (let arch in database) { - - let tree = Object.values(database[arch]); - if (tree.length > 0) { - - tree.filter((pkg) => { - return pkg.versions.length > 1; - }).forEach((pkg) => { - - pkg.versions.sort(sortByVersion).slice(1).forEach((version) => { - - remove_file(pkgs_folder + '/' + pkg.name + '-' + version + '-' + pkg.arch + '.pkg.tar.xz', (err) => { - if (!err) console.log(':: purged "' + pkg.name + '-' + version + '" (' + pkg.arch + ')'); - }); - - }); - - }); - - } - - } - - }); - -} else if (ACTION === 'download' && SERVER !== null) { - - let pkgs_folder = '/var/cache/pacman/pkg'; - let sync_folder = '/var/lib/pacman/sync'; - - if (FOLDER !== null) { - pkgs_folder = FOLDER + '/pkgs'; - sync_folder = FOLDER + '/sync'; - _mkdir(FOLDER + '/pkgs'); - _mkdir(FOLDER + '/sync'); - } - - - let on_download_complete = (pkgs_folder, packages) => { - - let upgrades = packages.filter((pkg) => pkg._success === true).map((pkg) => pkg.file); - if (upgrades.length > 0) { - console.log(''); - console.log(':: Use this to install upgrades from cache:'); - console.log(' cd "' + pkgs_folder + '"; sudo pacman -U ' + upgrades.join(' ') + ';'); - } - - let downloads = packages.filter((pkg) => pkg._success === false).map((pkg) => pkg.name); - if (downloads.length > 0) { - console.log(''); - console.log(':: Use this to download upgrades into cache:'); - console.log(' cd "' + pkgs_folder + '"; sudo pacman -Sw --cachedir "' + pkgs_folder + '" ' + downloads.join(' ') + ';'); - } - - }; - - - write_file('/tmp/pacman-backup.conf', toConfig(SERVER), (result) => { - - if (result === true) { - - let sync_handle = spawn('pacman', [ '-Sy', '--config', '/tmp/pacman-backup.conf' ], { - cwd: sync_folder - }); - - let stderr = (sync_handle.stderr || '').toString().trim(); - if (stderr.length === 0) { - - read_upgrades((upgrades) => { - - if (upgrades.length > 0) { - - read_packages(pkgs_folder, (cache) => { - - upgrades.forEach((pkg) => { - pkg._success = cache.includes(pkg.file); - }); - - - let downloads = upgrades.filter((pkg) => pkg._success === false); - if (downloads.length > 0) { - - downloads.forEach((pkg, d) => { - - download('http://' + SERVER + ':15678/' + pkg.file, (buffer) => { - - if (buffer !== null && buffer.length > 0) { - - write_file(pkgs_folder + '/' + pkg.file, buffer, (err) => { - - if (!err) console.log(':: downloaded ' + pkg.name + '-' + pkg.version); - pkg._success = true; - - if (d === downloads.length - 1) on_download_complete(pkgs_folder, upgrades); - - }); - - } else { - - pkg._success = false; - - if (d === downloads.length - 1) on_download_complete(pkgs_folder, upgrades); - - } - - }); - - }); - - } - - }); - - } - - }); - - } else { - - console.log(':! Cannot synchronize database with "' + SERVER + '".'); - console.log(stderr); - - process.exit(1); - - } - - } else { - - console.log(':! Cannot write to /tmp/pacman-backup.conf.'); - - process.exit(1); - - } - - }); - -} else if (ACTION === 'download') { - - let pkgs_folder = '/var/cache/pacman/pkg'; - - if (FOLDER !== null) { - pkgs_folder = FOLDER + '/pkgs'; - _mkdir(FOLDER + '/pkgs'); - } - - - read_upgrades((packages) => { - - if (packages.length > 0) { - - read_packages(pkgs_folder, (cache) => { - - let downloads = packages.filter((pkg) => cache.includes(pkg.file) === false); - if (downloads.length > 0) { - - console.log(''); - console.log(':: Copy/Paste this into a Download Manager of your choice:'); - console.log(''); - - downloads.forEach((pkg) => { - - let url = MIRROR; - if (url.endsWith('/')) { - url = url.substr(0, url.length - 1); - } - - url = url.replace('$arch', ARCH); - url = url.replace('$repo', pkg.repo); - - console.log(url + '/' + pkg.file); - - }); - - } - - }); - - } - - }); - -} else if (ACTION === 'serve') { - - let pkgs_folder = '/var/cache/pacman/pkg'; - let sync_folder = '/var/lib/pacman/sync'; - - if (FOLDER !== null) { - pkgs_folder = FOLDER + '/pkgs'; - sync_folder = FOLDER + '/sync'; - _mkdir(FOLDER + '/pkgs'); - _mkdir(FOLDER + '/sync'); - } - - - let database = []; - - fs.readdir(sync_folder, (err, files) => { - - if (!err) { - - files.filter((file) => { - return file.endsWith('.db'); - }).forEach((file) => { - database.push(file); - }); - - } - - }); - - - let server = http.createServer((req, res) => { - - let range = null; - - if (typeof req.headers.range === 'string') { - - let tmp = req.headers.range.replace('bytes=', '').split('/')[0].split('-'); - if (tmp.length === 2) { - - range = {}; - - let start = parseInt(tmp[0], 10); - let end = parseInt(tmp[1], 10); - - if (!isNaN(start)) range.start = start; - if (!isNaN(end)) range.end = end; - - if (typeof range.start !== 'number') { - range = null; - } - - } - - } - - - let file = req.url.split('/').pop(); - if (file.endsWith('.tar.xz') || file.endsWith('.tar.zst')) { - - if (range !== null) { - serve_with_range(pkgs_folder + '/' + file, range, res); - } else { - serve(pkgs_folder + '/' + file, res); - } - - } else if ((file.endsWith('.db') || file.endsWith('.db.sig')) && database.includes(file)) { - - if (range !== null) { - serve_with_range(sync_folder + '/' + file, range, res); - } else { - serve(sync_folder + '/' + file, res); - } - - } else { - - console.log(':! Cannot serve "' + file + '"'); - res.writeHead(404, {}); - res.end(); - - } - - }); - - server.on('error', (err) => { - - if (err.code === 'EADDRINUSE') { - - console.log(''); - console.log(':! pacman-backup is already serving on port 15678'); - - process.exit(1); - - } else { - - console.log(''); - console.log(':! ' + err.message); - - process.exit(1); - - } - - }); - - try { - server.listen(15678); - } catch (err) { - // Ignore - } - - - setTimeout(() => { - - let hostname = (spawn('hostname').stdout || '').toString().trim(); - let interfaces = Object.values(os.networkInterfaces()).flat().filter((iface) => iface.internal === false); - - if (hostname !== '') { - - console.log(''); - console.log(':: Use this to download from this machine:'); - console.log(' pacman-backup download ' + hostname + ';'); - - console.log(''); - console.log(':: Use this to override mirrorlist:'); - console.log(' sudo echo "Server = http://' + hostname + ':15678" > /etc/pacman.d/mirrorlist;'); - - } else if (interfaces.length > 0) { - - console.log(''); - console.log(':: Use this to download from this machine:'); - console.log(''); - - interfaces.forEach((iface) => { - console.log('pacman-backup download "' + iface.address + '";'); - }); - - } - - }, 250); - -} else if (ACTION === 'upgrade') { - - let pkgs_folder = '/var/cache/pacman/pkg'; - - if (FOLDER !== null) { - pkgs_folder = FOLDER + '/pkgs'; - _mkdir(FOLDER + '/pkgs'); - } - - - read_upgrades((packages) => { - - if (packages.length > 0) { - - read_packages(pkgs_folder, (cache) => { - - let upgrades = packages.filter((pkg) => cache.includes(pkg.file) === true); - if (upgrades.length > 0) { - console.log(''); - console.log(':: Use this to install upgrades from cache:'); - console.log(' cd "' + pkgs_folder + '"; sudo pacman -U ' + upgrades.map((pkg) => pkg.file).join(' ') + ';'); - } - - let downloads = packages.filter((pkg) => cache.includes(pkg.file) === false); - if (downloads.length > 0) { - console.log(''); - console.log(':: Use this to download upgrades into cache:'); - console.log(' cd "' + pkgs_folder + '"; sudo pacman -Sw --cachedir "' + pkgs_folder + '" ' + downloads.map((pkg) => pkg.name).join(' ') + ';'); - } - - }); - - } - - }); - -} else { - - console.log('pacman-backup'); - console.log('Backup tool for off-the-grid upgrades via portable USB drives or LAN networks.'); - console.log(''); - console.log('Usage: pacman-backup [Action] [Folder]'); - console.log(''); - console.log('Usage Notes:'); - console.log(''); - console.log(' If no folder is given, system-wide folders are used.'); - console.log(' If folder is given, it is assumed write-able by the current user.'); - console.log(' Remember to always "sync" after archive or cleanup.'); - console.log(''); - console.log('Available Actions:'); - console.log(''); - console.log(' archive copies local package cache to folder'); - console.log(' cleanup removes outdated packages from folder'); - console.log(' upgrade prints pacman command to upgrade from folder'); - console.log(''); - console.log(' serve serves packages as local network server'); - console.log(' download downloads packages to folder from server'); - console.log(''); - console.log(''); - console.log('USB Drive Example:'); - console.log(''); - console.log(' # Step 1: Machine with internet connection'); - console.log(''); - console.log(' sudo pacman -Sy;'); - console.log(' sudo pacman -Suw;'); - console.log(''); - console.log(' pacman-backup archive /run/media/' + USER + '/pacman-usbdrive;'); - console.log(' pacman-backup cleanup /run/media/' + USER + '/pacman-usbdrive;'); - console.log(''); - console.log(' sync;'); - console.log(''); - console.log(' # Step 2: ' + USER + ' walks to machine without internet connection and mounts same USB drive there ...'); - console.log(''); - console.log(' sudo cp /run/media/' + USER + '/pacman-usbdrive/sync/*.db /var/lib/pacman/sync/;'); - console.log(' pacman-backup upgrade /run/media/' + USER + '/pacman-usbdrive;'); - console.log(''); - console.log('LAN Server Example:'); - console.log(''); - console.log(' # 1: Machine with internet connection and example IP 192.168.0.10'); - console.log(''); - console.log(' sudo pacman -Sy;'); - console.log(' sudo pacman -Suw;'); - console.log(' pacman-backup serve;'); - console.log(''); - console.log(' # Step 2: User walks to machine with LAN connection to server...'); - console.log(''); - console.log(' # sudo echo "Server = http://192.168.0.10:15678" > /etc/pacman.d/mirrorlist'); - console.log(''); - console.log(' # Alternatively, use sudo pacman -Sy && sudo pacman -Suw;'); - console.log(' sudo pacman-backup download 192.168.0.10'); - console.log(' pacman-backup upgrade'); - console.log(''); - -} - diff --git a/source/actions/Serve.go b/source/actions/Serve.go index 47c8d07..fc3dae1 100644 --- a/source/actions/Serve.go +++ b/source/actions/Serve.go @@ -1,10 +1,263 @@ package actions +import "pacman-backup/pacman" +import "net/http" +import "os" +import "path/filepath" +import "strconv" +import "strings" + +func serveFileRange(response http.ResponseWriter, file string, start int64, end int64) { + + stat, err0 := os.Stat(file) + + if err0 == nil && !stat.IsDir() { + + last_modified := stat.ModTime().Format(http.TimeFormat) + + buffer, err1 := os.ReadFile(file) + + if err1 == nil { + + content_range := "bytes " + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end, 10) + "/" + strconv.FormatInt(stat.Size(), 10) + partial_buffer := buffer[start:end+1] + + header := response.Header() + header.Set("Content-Encoding", "identity") + header.Set("Content-Length", strconv.Itoa(len(partial_buffer))) + header.Set("Content-Range", content_range) + header.Set("Content-Type", "application/octet-stream") + header.Set("Date", last_modified) + header.Set("Last-Modified", last_modified) + + response.WriteHeader(http.StatusPartialContent) + response.Write(partial_buffer) + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + +} + +func serveFile(response http.ResponseWriter, file string) { + + stat, err0 := os.Stat(file) + + if err0 == nil && !stat.IsDir() { + + last_modified := stat.ModTime().Format(http.TimeFormat) + + buffer, err1 := os.ReadFile(file) + + if err1 == nil { + + header := response.Header() + header.Set("Accept-Ranges", "bytes") + header.Set("Content-Encoding", "identity") + header.Set("Content-Length", strconv.FormatInt(stat.Size(), 10)) + header.Set("Content-Type", "application/octet-stream") + header.Set("Date", last_modified) + header.Set("Last-Modified", last_modified) + + response.WriteHeader(http.StatusOK) + response.Write(buffer) + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + +} + +func serveFileHeader(response http.ResponseWriter, file string) { + + stat, err0 := os.Stat(file) + + if err0 == nil && !stat.IsDir() { + + last_modified := stat.ModTime().Format(http.TimeFormat) + + header := response.Header() + header.Set("Accept-Ranges", "bytes") + header.Set("Content-Encoding", "identity") + header.Set("Content-Length", strconv.FormatInt(stat.Size(), 10)) + header.Set("Content-Type", "application/octet-stream") + header.Set("Date", last_modified) + header.Set("Last-Modified", last_modified) + + response.WriteHeader(http.StatusOK) + response.Write([]byte{}) + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + +} + func Serve(sync_folder string, pkgs_folder string) bool { var result bool - // TODO: Implement Serve() method on port 15678 + http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) { + + path := request.URL.Path + file := filepath.Base(path) + + if request.Method == "GET" { + + range_header := strings.TrimSpace(request.Header.Get("Range")) + + if strings.HasPrefix(range_header, "bytes=") { + + tmp := strings.Split(range_header[6:], "-") + + if !strings.Contains(range_header, ",") && len(tmp) == 2 { + + // 0 is special end value to serve the rest of the file + if tmp[1] == "" { + tmp[1] = "0" + } + + start, err1 := strconv.ParseInt(tmp[0], 10, 64) + end, err2 := strconv.ParseInt(tmp[1], 10, 64) + + if err1 == nil && err2 == nil { + + if pacman.IsDatabaseFilename(file) { + serveFileRange(response, sync_folder + "/" + file, start, end) + } else if pacman.IsPackageFilename(file) { + serveFileRange(response, pkgs_folder + "/" + file, start, end) + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusBadRequest) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusBadRequest) + response.Write([]byte{}) + } + + } else { + + if_modified_since := strings.TrimSpace(request.Header.Get("If-Modified-Since")) + + if if_modified_since != "" { + + time, err0 := http.ParseTime(if_modified_since) + + if err0 == nil { + + if pacman.IsDatabaseFilename(file) { + + stat, err1 := os.Stat(sync_folder + "/" + file) + + if err1 == nil { + + if stat.ModTime().After(time) { + serveFile(response, sync_folder + "/" + file) + } else { + response.WriteHeader(http.StatusNotModified) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } else if pacman.IsPackageFilename(file) { + + stat, err1 := os.Stat(pkgs_folder + "/" + file) + + if err1 == nil { + + if stat.ModTime().After(time) { + serveFile(response, pkgs_folder + "/" + file) + } else { + response.WriteHeader(http.StatusNotModified) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } else { + + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + + } + + } else { + + if pacman.IsDatabaseFilename(file) { + serveFile(response, sync_folder + "/" + file) + } else if pacman.IsPackageFilename(file) { + serveFile(response, pkgs_folder + "/" + file) + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } + + } else { + + if pacman.IsDatabaseFilename(file) { + serveFile(response, sync_folder + "/" + file) + } else if pacman.IsPackageFilename(file) { + serveFile(response, pkgs_folder + "/" + file) + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } + + } + + } else if request.Method == "HEAD" { + + if pacman.IsDatabaseFilename(file) { + serveFileHeader(response, sync_folder + "/" + file) + } else if pacman.IsPackageFilename(file) { + serveFileHeader(response, pkgs_folder + "/" + file) + } else { + response.WriteHeader(http.StatusNotFound) + response.Write([]byte{}) + } + + } else { + response.WriteHeader(http.StatusMethodNotAllowed) + response.Write([]byte{}) + } + + }) + + err := http.ListenAndServe(":15678", nil) + + if err == nil { + result = true + } return result diff --git a/source/cmds/pacman-backup/main.go b/source/cmds/pacman-backup/main.go index 837b9b1..4385a39 100644 --- a/source/cmds/pacman-backup/main.go +++ b/source/cmds/pacman-backup/main.go @@ -181,6 +181,23 @@ func main() { actions.Sync(mirror, config.Options.DBPath + "/sync", config.Options.CacheDir) actions.Download(mirror, config.Options.DBPath + "/sync", config.Options.CacheDir) + // pacman-backup download /mnt/usb-drive + } else if isFolder(os.Args[2]) { + + config := pacman.InitConfig() + mirror := config.ToMirror() + + if !isFolder(os.Args[2] + "/sync") { + makeFolder(os.Args[2] + "/sync") + } + + if !isFolder(os.Args[2] + "/pkgs") { + makeFolder(os.Args[2] + "/pkgs") + } + + actions.Sync(mirror, os.Args[2] + "/sync", os.Args[2] + "/pkgs") + actions.Download(mirror, os.Args[2] + "/sync", os.Args[2] + "/pkgs") + } } else if action == "import" { diff --git a/source/pacman/Download.go b/source/pacman/Download.go index b100852..71e7413 100644 --- a/source/pacman/Download.go +++ b/source/pacman/Download.go @@ -7,7 +7,8 @@ func Download(config string, name string) bool { var result bool - cmd := exec.Command("pacman", "-Sw", "--noconfirm", "--config", config, name) + // Download without dependency checks for better UI + cmd := exec.Command("pacman", "-Swdd", "--noconfirm", "--config", config, name) err := cmd.Run() if err == nil { diff --git a/source/pacman/IsDatabaseFilename.go b/source/pacman/IsDatabaseFilename.go index dd074a6..9c633b3 100644 --- a/source/pacman/IsDatabaseFilename.go +++ b/source/pacman/IsDatabaseFilename.go @@ -16,6 +16,8 @@ func IsDatabaseFilename(filepath string) bool { if strings.HasSuffix(filename, ".db") { result = true + } else if strings.HasSuffix(filename, ".db.sig") { + result = true } else if strings.HasSuffix(filename, ".files") { result = true }