Consider upgrading the LMDB library

[leiless]

https://github.com/LMDB/lmdb/blob/mdb.master/libraries/liblmdb/lmdb.h#L227-L232

The upstream LMDB is already 0.9.70 and has many bug fixes.
py-lmdb binding is currently 0.9.29 (Mar 17, 2021) (This version does not support sparse DB file on Windows).

Please consider upgrading the LMDB lib source code.

[leiless]

FYI, I happened to bump the LMDB lib version to 0.9.70 (based on the HEAD commit 30288c72573ceac719627183f1058cad1dd08b74) and rebuild the lib/py-lmdb/env-copy-txn.patch.

The py-lmdb package was successfully build on Windows 11 and Ubuntu 22.04 LTS.

$ pip3.exe install -U patch-ng
$ python.exe ./setup.py build
...
cl : Command line warning D9025 : overriding '/DNDEBUG' with '/UNDEBUG'
cl : Command line warning D9025 : overriding '/W3' with '/w'
cpython.c
"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.38.33130\bin\HostX86\x64\link.exe" /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:C:\Python311\libs /LIBPATH:C:\Python311 /LIBPATH:C:\Python311\PCbuild\amd64 "/LIBPATH:C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.38.33130\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.22000.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\\lib\10.0.22000.0\\um\x64" Advapi32.lib /EXPORT:PyInit_cpython build\temp.win-amd64-cpython-311\Release\build/lib/mdb.obj build\temp.win-amd64-cpython-311\Release\build/lib/midl.obj build\temp.win-amd64-cpython-311\Release\lmdb/cpython.obj /OUT:build\lib.win-amd64-cpython-311\lmdb\cpython.cp311-win_amd64.pyd /IMPLIB:build\temp.win-amd64-cpython-311\Release\build/lib\cpython.cp311-win_amd64.lib
   Creating library build\temp.win-amd64-cpython-311\Release\build/lib\cpython.cp311-win_amd64.lib and object build\temp.win-amd64-cpython-311\Release\build/lib\cpython.cp311-win_amd64.exp
Generating code
Finished generating code

>>> import lmdb
>>> lmdb.version(True)
(0, 9, 70, 1)

Don't know if any tests can be carried on?

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 28cfc36f53..5b32750031 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -722,9 +722,13 @@ int  mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
 	 *		consumes more CPU and runs more slowly than the default.
 	 *		Currently it fails if the environment has suffered a page leak.
 	 * </ul>
+	 * @param[in] txn Transaction used for the copy.  If NULL, a temporary
+	 * transaction will be used.  This is only valid if the #MDB_CP_COMPACT
+	 * flag is set.
+	 *
 	 * @return A non-zero error value on failure and 0 on success.
 	 */
-int  mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
+int  mdb_env_copy3(MDB_env *env, const char *path, unsigned int flags, MDB_txn *txn);
 
 	/** @brief Copy an LMDB environment to the specified file descriptor,
 	 *	with options.
@@ -741,9 +745,12 @@ int  mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
 	 * have already been opened for Write access.
 	 * @param[in] flags Special options for this operation.
 	 * See #mdb_env_copy2() for options.
+	 * @param[in] txn Transaction used for the copy.  If NULL, a temporary
+	 * transaction will be used.  This is only valid if the #MDB_CP_COMPACT
+	 * flag is set.
 	 * @return A non-zero error value on failure and 0 on success.
 	 */
-int  mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags);
+int  mdb_env_copyfd3(MDB_env *env, mdb_filehandle_t fd, unsigned int flags, MDB_txn *txn);
 
 	/** @brief Return statistics about the LMDB environment.
 	 *
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 965ba2a2b8..f32a681baa 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10375,12 +10375,12 @@ done:
 
 	/** Copy environment with compaction. */
 static int ESECT
-mdb_env_copyfd1(MDB_env *env, HANDLE fd)
+mdb_env_copyfd1(MDB_env *env, HANDLE fd, MDB_txn *txn)
 {
 	MDB_meta *mm;
 	MDB_page *mp;
 	mdb_copy my = {0};
-	MDB_txn *txn = NULL;
+	MDB_txn *orig_txn = txn;
 	pthread_t thr;
 	pgno_t root, new_root;
 	int rc = MDB_SUCCESS;
@@ -10426,9 +10426,11 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 	if (rc)
 		goto done;
 
-	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
-	if (rc)
-		goto finish;
+	if (!txn) {
+		rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+		if (rc)
+			goto finish;
+	}
 
 	mp = (MDB_page *)my.mc_wbuf[0];
 	memset(mp, 0, NUM_METAS * env->me_psize);
@@ -10488,7 +10490,8 @@ finish:
 		my.mc_error = rc;
 	mdb_env_cthr_toggle(&my, 1 | MDB_EOF);
 	rc = THREAD_FINISH(thr);
-	mdb_txn_abort(txn);
+	if (!orig_txn)
+		mdb_txn_abort(txn);
 
 done:
 #ifdef _WIN32
@@ -10605,12 +10608,22 @@ leave:
 }
 
 int ESECT
-mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
+mdb_env_copyfd3(MDB_env *env, HANDLE fd, unsigned int flags, MDB_txn *txn)
 {
 	if (flags & MDB_CP_COMPACT)
-		return mdb_env_copyfd1(env, fd);
+		return mdb_env_copyfd1(env, fd, txn);
 	else
+	{
+		if (txn) /* may only use txn with compact */
+			return EINVAL;
 		return mdb_env_copyfd0(env, fd);
+	}
+}
+
+int ESECT
+mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
+{
+	return mdb_env_copyfd3(env, fd, flags, NULL);
 }
 
 int ESECT
@@ -10621,6 +10634,12 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd)
 
 int ESECT
 mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
+{
+	return mdb_env_copy3(env, path, flags, NULL);
+}
+
+int ESECT
+mdb_env_copy3(MDB_env *env, const char *path, unsigned int flags, MDB_txn *txn)
 {
 	int rc;
 	MDB_name fname;
@@ -10632,7 +10651,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 		mdb_fname_destroy(fname);
 	}
 	if (rc == MDB_SUCCESS) {
-		rc = mdb_env_copyfd2(env, newfd, flags);
+		rc = mdb_env_copyfd3(env, newfd, flags, txn);
 		if (close(newfd) < 0 && rc == MDB_SUCCESS)
 			rc = ErrCode();
 	}

[ghost]

@leiless any chance you put a windows build on pypi?

[jnwatson]

Working on it. #361