Merge pull request #562 from evo-lua/488-vfs-dlopen-dlname

Implement VFS functions to load shared libraries from within LUAZIP archives

Runtime/Libraries/vfs.lua

@@ -1,5 +1,6 @@
 local ffi = require("ffi")
 local miniz = require("miniz")
+local uv = require("uv")
 local validation = require("validation")
 
 local ffi_cast = ffi.cast
@@ -22,6 +23,10 @@ local vfs = {
 		} lua_zip_signature_t;
 		#pragma pack(pop)
 	]],
+	cachedAppBundles = {},
+	errorStrings = {
+		MISSING_APP_BUNDLE = "Not a LUAZIP app bundle",
+	},
 }
 
 function vfs.decode(fileContents)
@@ -102,6 +107,52 @@ function vfs.searcher(zipApp, moduleName)
 	end
 end
 
+function vfs.dlopen(zipApp, libraryName)
+	if not zipApp then
+		-- Ensure it's a NOOP if run from a regular script file that may also be bundled
+		return nil, vfs.errorStrings.MISSING_APP_BUNDLE
+	end
+
+	validation.validateString(libraryName, "libraryName")
+
+	libraryName = vfs.dlname(libraryName)
+	local fileContents, errorMessage = vfs.extract(zipApp, libraryName)
+	if not fileContents then
+		return nil, errorMessage
+	end
+
+	local tempDirectoryPath = uv.fs_mkdtemp(path.join(uv.cwd(), "LUAZIP-XXXXXX"))
+	local tempFilePath = path.join(tempDirectoryPath, libraryName)
+	C_FileSystem.WriteFile(tempFilePath, fileContents)
+	local success, libraryOrErrorMessage = pcall(ffi.load, tempFilePath)
+
+	-- Cleanup should never fail, but if it does at least it'll do so loudly
+	assert(C_FileSystem.Delete(tempFilePath))
+	assert(C_FileSystem.Delete(tempDirectoryPath))
+
+	if not success then
+		return nil, libraryOrErrorMessage
+	end
+
+	return libraryOrErrorMessage
+end
+
+function vfs.dlname(libraryName)
+	validation.validateString(libraryName, "libraryName")
+
+	-- AFAICT there's no reason to support .dylib on macOS? For now, should be safe to assume .so
+	local extension = string.lower(path.extname(libraryName))
+	if extension == ".dll" or extension == ".so" then
+		return libraryName
+	end
+
+	if ffi.os == "Windows" then
+		return libraryName .. ".dll"
+	else
+		return "lib" .. libraryName .. ".so"
+	end
+end
+
 ffi.cdef(vfs.cdefs)
 
 return vfs

Runtime/evo.lua

@@ -146,6 +146,9 @@ function evo.run()
 		-- For security and performance reasons, prefer loading from the VFS in case of conflicts
 		table.insert(package.searchers, 1, vfsSearcher)
 
+		-- Avoid having to decode archives multiple times (the design could use some refinement)
+		vfs.cachedAppBundles[uv.exepath()] = zipApp
+
 		return vfs.dofile(zipApp, evo.DEFAULT_ENTRY_POINT)
 	end
 

Tests/BDD/vfs-library.spec.lua

@@ -1,3 +1,4 @@
+local ffi = require("ffi")
 local vfs = require("vfs")
 
 describe("vfs", function()
@@ -103,4 +104,46 @@ describe("vfs", function()
 			assertEquals(vfs.extract(zipApp, vfsPath), expectedFileContents)
 		end)
 	end)
+
+	describe("dlname", function()
+		it("should throw if an invalid library name was passed", function()
+			assertThrows(function()
+				vfs.dlname(nil)
+			end, "Expected argument libraryName to be a string value, but received a nil value instead")
+		end)
+
+		it("should return the input string if the library name indicates a Windows DLL", function()
+			assertEquals(vfs.dlname("foo.dll"), "foo.dll")
+			assertEquals(vfs.dlname("some/directory/foo.dll"), "some/directory/foo.dll")
+		end)
+
+		it("should return the input string if the library name indicates a shared object file", function()
+			assertEquals(vfs.dlname("libfoo.so"), "libfoo.so")
+			assertEquals(vfs.dlname("some/directory/libfoo.so"), "some/directory/libfoo.so")
+		end)
+
+		it("should be able to recognize valid extensions with inconsistent capitalization", function()
+			assertEquals(vfs.dlname("libfoo.SO"), "libfoo.SO")
+			assertEquals(vfs.dlname("foo.dLL"), "foo.dLL")
+		end)
+
+		it("should adhere to platform-specific conventions if the library name isn't fully qualified", function()
+			local isWindows = ffi.os == "Windows"
+			assertEquals(vfs.dlname("foo"), isWindows and "foo.dll" or "libfoo.so")
+		end)
+	end)
+
+	describe("dlopen", function()
+		it("should fail if no app bundle was provided", function()
+			assertFailure(function()
+				return vfs.dlopen(nil, "foo")
+			end, vfs.errorStrings.MISSING_APP_BUNDLE)
+		end)
+
+		it("should throw if an invalid library name was passed", function()
+			assertThrows(function()
+				vfs.dlopen({}, nil)
+			end, "Expected argument libraryName to be a string value, but received a nil value instead")
+		end)
+	end)
 end)

Tests/Fixtures/dlopen-test-app/main.lua

@@ -0,0 +1,41 @@
+local assertions = require("assertions")
+local ffi = require("ffi")
+local uv = require("uv")
+local vfs = require("vfs")
+local zlibStatic = require("zlib")
+
+local assertEquals = assertions.assertEquals
+local assertTrue = assertions.assertTrue
+
+-- REMINDER: This part is copy/pasted from vfs-dlopen-zlib test (remove it later)
+local testAppDirectory = path.join("Tests", "Fixtures", "dlopen-test-app")
+-- If zlib hasn't been built, the test won't work (unfortunate, but acceptable for now)
+local sharedLibraryExtension = (ffi.os == "Windows" and "dll" or "so")
+local sharedLibraryPath = path.join(testAppDirectory, "zlib." .. sharedLibraryExtension)
+assertTrue(C_FileSystem.Exists(sharedLibraryPath))
+
+-- Basic sanity check: Let's first make sure that the shared object can be loaded at all
+-- Note: The defined symbols must not clash in case FFI bindings for zlib also exist
+local cdefs = [[
+	const char* zlibVersion(void);
+]] -- If these symbols are already part of the zlib bindings, can probably remove this
+
+ffi.cdef(cdefs)
+local sharedLibrary = ffi.load(sharedLibraryPath)
+assertEquals(type(sharedLibrary), "userdata")
+local ffiVersion = sharedLibrary.zlibVersion()
+assertEquals(type(ffiVersion), "cdata")
+local ffiVersionString = ffi.string(ffiVersion)
+assertEquals(type(ffiVersionString), "string")
+
+-- End of copy/pasted part
+local zipApp = vfs.cachedAppBundles[uv.exepath()]
+local zlibShared = assert(vfs.dlopen(zipApp, "zlib.so"))
+local versionString = ffi.string(zlibShared.zlibVersion())
+
+local zlibVersionMajor, zlibVersionMinor, zlibVersionPatch = zlibStatic.version()
+local semanticZlibVersionString = format("%d.%d.%d", zlibVersionMajor, zlibVersionMinor, zlibVersionPatch or 0)
+
+printf("Shared ZLIB version: %s", versionString)
+printf("Static ZLIB version: %s", semanticZlibVersionString)
+assertEquals(versionString, semanticZlibVersionString)

Tests/Integration/vfs-dlopen-zlib.lua

@@ -0,0 +1,47 @@
+local assertions = require("assertions")
+local evo = require("evo")
+local ffi = require("ffi")
+
+local assertEquals = assertions.assertEquals
+local assertFalse = assertions.assertFalse
+local assertTrue = assertions.assertTrue
+
+-- REMINDER: This part is copy/pasted into the actual test app (remove it later)
+local testAppDirectory = path.join("Tests", "Fixtures", "dlopen-test-app")
+-- If zlib hasn't been built, the test won't work (unfortunate, but acceptable for now)
+local sharedLibraryExtension = (ffi.os == "Windows" and "dll" or "so")
+local sharedLibraryPath = path.join(testAppDirectory, "zlib." .. sharedLibraryExtension)
+assertTrue(C_FileSystem.Exists(sharedLibraryPath))
+
+-- Basic sanity check: Let's first make sure that the shared object can be loaded at all
+-- Note: The defined symbols must not clash in case FFI bindings for zlib also exist
+local cdefs = [[
+	const char* zlibVersion(void);
+]] -- If these symbols are already part of the zlib bindings, can probably remove this
+
+ffi.cdef(cdefs)
+local sharedLibrary = ffi.load(sharedLibraryPath)
+assertEquals(type(sharedLibrary), "userdata")
+local ffiVersion = sharedLibrary.zlibVersion()
+assertEquals(type(ffiVersion), "cdata")
+local ffiVersionString = ffi.string(ffiVersion)
+assertEquals(type(ffiVersionString), "string")
+
+-- Now, a new LUAZIP app is needed since finalized miniz archives can't be diff-patched ...
+-- This should probably be a separate library function in the vfs module, but right now it isn't
+evo.buildZipApp("build", { testAppDirectory })
+assertTrue(C_FileSystem.Exists("dlopen-test-app"))
+assertTrue(C_FileSystem.Exists("dlopen-test-app.zip"))
+
+-- Running the test with os.execute should be enough here as it mimicks real apps
+local EXIT_SUCCESS = 0
+local status, terminationReason, exitCode = os.execute("./dlopen-test-app")
+assertTrue(status)
+assertEquals(terminationReason, "exit")
+assertEquals(exitCode, EXIT_SUCCESS)
+
+-- Cleanup: Ideally this would be automatically handled by the interpreter CLI
+C_FileSystem.Delete("dlopen-test-app")
+C_FileSystem.Delete("dlopen-test-app.zip")
+assertFalse(C_FileSystem.Exists("dlopen-test-app"))
+assertFalse(C_FileSystem.Exists("dlopen-test-app.zip"))

deps/zlib-unixbuild.sh

@@ -6,8 +6,12 @@ OUT_DIR=ninjabuild-unix
 SRC_DIR=deps/madler/zlib
 BUILD_DIR=$SRC_DIR/cmakebuild-unix
 
-cmake -S $SRC_DIR -B $BUILD_DIR -G Ninja -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_COMPILER=gcc
+cmake -S $SRC_DIR -B $BUILD_DIR -G Ninja -DBUILD_SHARED_LIBS=ON -DCMAKE_C_COMPILER=gcc
 cmake --build $BUILD_DIR --clean-first
 
 cp $BUILD_DIR/libz.a $OUT_DIR/zlibstatic.a
-cp $BUILD_DIR/zconf.h $OUT_DIR
+cp $BUILD_DIR/zconf.h $OUT_DIR
+
+# The shared library version is only used as a test fixture (somewhat arbitrary choice)
+TEST_APP_DIR=$(pwd)/Tests/Fixtures/dlopen-test-app
+cp $BUILD_DIR/libz.so $TEST_APP_DIR/zlib.so || cp $BUILD_DIR/libz.dylib $TEST_APP_DIR/zlib.so

deps/zlib-windowsbuild.sh

@@ -6,8 +6,12 @@ OUT_DIR=ninjabuild-windows
 SRC_DIR=deps/madler/zlib
 BUILD_DIR=$SRC_DIR/cmakebuild-windows
 
-cmake -S $SRC_DIR -B $BUILD_DIR -G Ninja -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_COMPILER=gcc
+cmake -S $SRC_DIR -B $BUILD_DIR -G Ninja -DBUILD_SHARED_LIBS=ON -DCMAKE_C_COMPILER=gcc
 cmake --build $BUILD_DIR --clean-first
 
 cp $BUILD_DIR/libzlibstatic.a $OUT_DIR/zlibstatic.a
-cp $BUILD_DIR/zconf.h $OUT_DIR
+cp $BUILD_DIR/zconf.h $OUT_DIR
+
+# The shared library version is only used as a test fixture (somewhat arbitrary choice)
+TEST_APP_DIR=$(pwd)/Tests/Fixtures/dlopen-test-app
+cp $BUILD_DIR/libzlib.dll $TEST_APP_DIR/zlib.dll || cp $BUILD_DIR/libzlib1.dll $TEST_APP_DIR/zlib.dll