fixed: PackageManager hangs on Python 2 (Venus OS prior to v2.80)

updatePackage: rewrite update file sets loop for speed improvement
replace OS's patch with one that handles more cases (eg, 0 context) and options
fixed: PackageManager hangs on Python 2 (Venus OS prior to v2.80)

HelperResources/CommonResources

@@ -36,6 +36,9 @@ restartGenerator=false
 restartSystemCalc=false
 restartDigitalinputs=false
 
+# use local patch executable - BusyBox version has bugs
+patch="/data/SetupHelper/patch"
+
 
 # file lists are populated by getFileLists called from_chckFileSets and autoinstall
 #	so these are global
@@ -686,8 +689,8 @@ restoreActiveFile ()
 		#	if success, don't restore original
 		previousPatchFile="$previousPatchesDir/$baseName.patch"
 		if [ -e "$previousPatchFile" ] && 
-			yes 'no' | patch -R "$activeFile" "$previousPatchFile" &> /dev/null ; then
-				restoreOriginal=false
+			$patch -f -s --reverse --reject-file=/dev/null "$activeFile" "$previousPatchFile" &> /dev/null ; then
+			restoreOriginal=false
 		# no previous patch file or reverse patch failed
 		#	original will be restored
 		else
@@ -1577,7 +1580,6 @@ if ! [ -z "$fileListPatched" ];then
 	for file in ${fileListPatched[@]}; do
 		baseName=$( basename $file )
 		tempActiveFile="$tempFileDir/$baseName"
-
 		if [ -e "$file" ] ; then
 			previousPatchFile="$previousPatchesDir/$baseName.patch"
 
@@ -1596,65 +1598,63 @@ if ! [ -z "$fileListPatched" ];then
 				done
 			fi
 
-			cp "$file" "$tempActiveFile"
 			patchOk=false
-			# package not currently in .package list - patch active file
-			if ! $thisPackageInList; then
+			# no other packages in list - ignore any previous patch and patch .orig file
+			if ! $otherPackagesInList && $thisPackageInList && [ -e "$file.orig" ]; then
+					cp "$file.orig" "$tempActiveFile"
+					patchOk=true
+			# this package not currently in .package list - patch active file
+			elif ! $thisPackageInList; then
+				cp "$file" "$tempActiveFile"
 				patchOk=true
 			# attempt to remove the previous patch for this package
 			# if this succeeds proceed with forward patch on result
-			elif [ -e "$previousPatchFile" ]; then
-				if yes 'no' | patch -R "$tempActiveFile" "$previousPatchFile" &> /dev/null ; then
+			elif $thisPackageInList && [ -e "$previousPatchFile" ]; then
+				if $patch -f -s --reverse --reject-file=/dev/null "$tempActiveFile" "$previousPatchFile" &> /dev/null ; then
+					cp "$file" "$tempActiveFile"
 					patchOk=true
 				# reverse patch failed
 				else
 					patchErrors+=( "unable to remove previous patch for $baseName" )
 					rm "$previousPatchFile"
 				fi
-			# no previous patch file
-			# active file prpbably modified by direct replaceent by this package
-			# patch .orig in stead of active file
-			elif ! $otherPackagesInList && [ -e "$file.orig" ]; then
-				cp "$file.orig" "$tempActiveFile"
-				patchOk=true
-			elif ! [ -e "$file.orig" ]; then
-				patchErrors+=( "$baseName no original for patch" )
-			else
-				patchErrors+=( "$baseName modified by other packages - and no previous patch for $packageName" )
 			fi
-
+			patchSuccess=false
+			forwardPatched="$tempFileDir/$baseName".patchedForInstall
+			currentPatchFile="$tempFileDir/$baseName.currentPatch"
+			reversePatchTest="$tempFileDir/$baseName.reversePatchTest"
 			# a suitable source for the patch was located above
 			if $patchOk; then
 				# attempt to patch the active file with any file ending in .patch
 				# the first one that successfully creates a forward AND reverse patch
 				# .patchedForInstall provides the patched file for updateActiveFile
-				forwardPatched="$tempFileDir/$baseName".patchedForInstall
-				reversePatchTest="$tempFileDir/$baseName.reversePatchTest"
 				patchFiles=( $( ls "$patchSourceDir/$baseName"*.patch ) )
-				patchFound=false
 				for patchFile in ${patchFiles[@]};do
 					cp "$tempActiveFile" "$forwardPatched"
-					if yes 'no' | patch -N "$forwardPatched" "$patchFile" &> /dev/null ; then
+					if $patch -f -s --forward --reject-file=/dev/null "$forwardPatched" "$patchFile" &> /dev/null ; then
 						# forward patch succeeded - test reverse patch (both must succeed)
 						cp "$forwardPatched" "$reversePatchTest"
-						if yes 'no' | patch -R "$reversePatchTest" "$patchFile" &> /dev/null ; then
-							patchFound=true
+						if $patch -f -s --reverse --reject-file=/dev/null "$reversePatchTest" "$patchFile" &> /dev/null ; then
+							patchSuccess=true
 							break
 						fi
 					fi
 				done
-				currentPatchFile="$tempFileDir/$baseName.currentPatch"
-				if $patchFound ; then
+				if $patchSuccess ; then
 					# save this so patch file used for install can be saved for evenutal uninstall
 					#	or to reverse patch prior to reinstall
 					cp "$patchFile" "$currentPatchFile"
 				else
 					patchErrors+=( "patch unsuccesful for $baseName" )
-					rm -r "$forwardPatched"
-					rm -f "$currentPatchFile"
 				fi
+			else
+				patchErrors+=( "no patch source for $baseName" )
+			fi
+			rm -f "$reversePatchTest"
+			if ! $patchSuccess ; then
+				rm -f "$forwardPatched"
+				rm -f "$currentPatchFile"
 			fi
-			rm -f "$reversePatched" "$reversePatchTest"
 		else
 			patchErrors+=( "$baseName no active file for patch" )
 			continue

HelperResources/version

@@ -1 +1 @@
-v8.0~24
+v8.0~25

PackageManager.py

@@ -366,8 +366,13 @@
 import re
 import glob
 
-PythonVersion = sys.version_info
 # accommodate both Python 2 (prior to v2.80) and 3
+# note subprocess.run and subprocess.DEVNULL do not exist in python 2
+#	so subprocess.Popen and subprocess.PIPE and subprocess.communicate ()
+#	are used in all subprodess calls even if process output is not needed
+#	or if it is not necessary to wait for the command to finish
+
+PythonVersion = sys.version_info
 if PythonVersion >= (3, 0):
 	import queue
 	from gi.repository import GLib
@@ -891,21 +896,18 @@ def RemoveDbusSettings (cls, settingsList):
 
 		# remove the dbus Settings paths - via the command line 
 		try:
-			proc = subprocess.Popen (['dbus', '-y', 'com.victronenergy.settings', '/', 'RemoveSettings', settingsToRemove  ],
-						stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			proc = subprocess.Popen (['dbus', '-y', 'com.victronenergy.settings', '/',
+						'RemoveSettings', settingsToRemove  ],
+						bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			_, stderr = proc.communicate ()
+			stderr = stderr.decode ().strip ()
+			returnCode = proc.returncode
 		except:
 			logging.error ("dbus RemoveSettings call failed")
 		else:
-			proc.wait()
-			# convert from binary to string
-			out, err = proc.communicate ()
-			stdout = out.decode ().strip ()
-			stderr = err.decode ().strip ()
-			returnCode = proc.returncode
 			if returnCode != 0:
 				logging.error ("dbus RemoveSettings failed " + str (returnCode))
 				logging.error ("stderr: " + stderr)
-
 
 	#	UpdateStatus
 	#
@@ -2172,17 +2174,14 @@ def updateGitHubVersion (self, packageName, gitHubUser, gitHubBranch):
 		url = "https://raw.githubusercontent.com/" + gitHubUser + "/" + packageName + "/" + gitHubBranch + "/version"
 		try:
 			proc = subprocess.Popen (["wget", "--timeout=10", "-qO", "-", url],
-							stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+						bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			stdout, _ = proc.communicate ()
+			stdout = stdout.decode ().strip ()
+			returnCode = proc.returncode
 		except:
 			logging.error ("wget for version failed " + packageName)
 			gitHubVersion = ""
 		else:
-			proc.wait()
-			# convert from binary to string
-			stdout, stderr = proc.communicate ()
-			stdout = stdout.decode ().strip ()
-			stderr = stderr.decode ().strip ()
-			returnCode = proc.returncode
 			if proc.returncode == 0:
 				gitHubVersion = stdout
 			else:
@@ -2456,32 +2455,30 @@ def GitHubDownload (self, packageName= None, source=None):
 
 		url = "https://github.com/" + gitHubUser + "/" + packageName  + "/archive/" + gitHubBranch  + ".tar.gz"
 		try:
-			proc = subprocess.Popen ( ['wget', '--timeout=120', '-qO', tempArchiveFile, url ],\
-										stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+			proc = subprocess.Popen ( ['wget', '--timeout=120', '-qO', tempArchiveFile, url ],
+								bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
+			_, stderr = proc.communicate()
+			stderr = stderr.decode ().strip ()
+			returnCode = proc.returncode
 		except:
 			errorMessage = "could not access archive on GitHub " + packageName
 			downloadError = True
 		else:
-			proc.wait()
-			returnCode = proc.returncode
 			if returnCode != 0:
 				errorMessage = "could not access " + packageName + ' ' + gitHubUser + ' ' + gitHubBranch + " on GitHub"
 				errorDetails = "returnCode" + str (returnCode) +  "stderr" + stderr
 				downloadError = True
 		if not downloadError:
 			try:
 				proc = subprocess.Popen ( ['tar', '-xzf', tempArchiveFile, '-C', tempDirectory ],
-											stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+								bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+				_, stderr = proc.communicate ()
+				stderr = stderr.decode ().strip ()
+				returnCode = proc.returncode
 			except:
 				errorMessage = "could not unpack " + packageName + ' ' + gitHubUser + ' ' + gitHubBranch
 				downloadError = True
 			else:
-				proc.wait()
-				stdout, stderr = proc.communicate ()
-				# convert from binary to string
-				stdout = stdout.decode ().strip ()
-				stderr = stderr.decode ().strip ()
-				returnCode = proc.returncode
 				if returnCode != 0:
 					errorMessage = "unpack failed " + packageName + ' ' + gitHubUser + ' ' + gitHubBranch
 					errorDetails = "stderr: " + stderr
@@ -2766,12 +2763,9 @@ def InstallPackage ( self, packageName=None, source=None , action='install' ):
 		DbusIf.UpdateStatus ( message=action + "ing " + packageName, where=sendStatusTo, logLevel=WARNING )
 		try:
 			proc = subprocess.Popen ( [ setupFile, action, 'runFromPm' ],
-										stdout=subprocess.PIPE, stderr=subprocess.PIPE )
-			proc.wait()
-			# convert from binary to string
-			out, err = proc.communicate ()
-			stdout = out.decode ().strip ()
-			stderr = err.decode ().strip ()
+										bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
+			_, stderr = proc.communicate ()
+			stderr = stderr.decode ().strip ()
 			returnCode = proc.returncode
 			setupRunFail = False
 		except:
@@ -3053,19 +3047,16 @@ def transferPackage (self, path, autoInstallOverride=False):
 		# unpack the archive - result is placed in tempDirectory
 		try:
 			proc = subprocess.Popen ( ['tar', '-xzf', path, '-C', tempDirectory ],
-										stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+							bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			_, stderr = proc.communicate ()
+			stderr = stderr.decode ().strip ()
+			returnCode = proc.returncode
 		except:
 			DbusIf.UpdateStatus ( message="tar failed for " + packageName,
 									where='Media', logLevel=ERROR)
 			time.sleep (5.0)
 			DbusIf.UpdateStatus ( message="", where='Media')
 			return False
-		proc.wait()
-		stdout, stderr = proc.communicate ()
-		# convert from binary to string
-		stdout = stdout.decode ().strip ()
-		stderr = stderr.decode ().strip ()
-		returnCode = proc.returncode
 		if returnCode != 0:
 			DbusIf.UpdateStatus ( message="could not unpack " + packageName + " from SD/USB media",
 									where='Media', logLevel=ERROR)
@@ -3243,8 +3234,8 @@ def settingsBackup (self, backupPath, settingsOnly = False):
 					shutil.rmtree (logDestDir)
 
 				proc = subprocess.Popen ( [ 'zip', '-rq', backupPath + "/logs.zip", "/data/log" ],
-											stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL )
-				proc.wait()
+										bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
+				proc.commiunicate()	#output ignored
 				returnCode = proc.returncode
 				logsWritten = "logs"
 			except:
@@ -3324,9 +3315,10 @@ def settingsRestore (self, backupPath, settingsOnly = False):
 							method = 'AddSetting'
 
 						try:
-							subprocess.run ( [ 'dbus', '-y', 'com.victronenergy.settings', '/', method, '',
+							proc = subprocess.Popen ( [ 'dbus', '-y', 'com.victronenergy.settings', '/', method, '',
 											path, default, typeId, min, max ], 
-											stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+											bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+							proc.commiunicate ()	# output ignored
 							parameterExists = True
 							logging.warning ("settingsRestore: creating " + path)
 						except:
@@ -3706,7 +3698,12 @@ def mainLoop ():
 	# an old RTC
 	timeSyncCommand = '/etc/init.d/save-rtc.sh'
 	if startTime > lastTimeSync + 30 and os.path.exists (timeSyncCommand):
-		subprocess.run ( [ timeSyncCommand ] )
+		try:
+			subprocess.Popen ( [ timeSyncCommand ],
+					bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			proc.commiunicate ()	# output ignored
+		except:
+			pass
 		lastTimeSync = startTime
 
 	packageName = "none"
@@ -3950,15 +3947,17 @@ def	directUninstall (packageName):
 	try:
 		setupFile = "/data/" + packageName + "/setup"
 		if os.path.isfile(setupFile)and os.access(setupFile, os.X_OK):
-			proc = subprocess.run ( [ setupFile, 'uninstall', 'runFromPm' ] )
-			proc.wait()
+			proc = subprocess.Popen ( [ setupFile, 'uninstall', 'runFromPm' ],
+						bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			proc.commiunicate ()	# output ignored
 			returnCode = proc.returncode
-			if returnCode == EXIT_REBOOT:
-				SystemReboot = True
-			elif returnCode == EXIT_RESTART_GUI:
-				GuiRestart = True
 	except:
 		logging.critical ("could not uninstall " + packageName)
+	else:
+		if returnCode == EXIT_REBOOT:
+			SystemReboot = True
+		elif returnCode == EXIT_RESTART_GUI:
+			GuiRestart = True
 
 
 #	main
@@ -4219,8 +4218,12 @@ def main():
 			command.append ("guiRestart")
 
 		# this runs in the background and will CONTINUE after PackageManager.py exits below
-		logging.critical ("finishing up in packageManagerEnd.sh")
-		subprocess.Popen ( command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL )
+		try:
+			logging.critical ("finishing up in packageManagerEnd.sh")
+			proc = subprocess.Popen ( command, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
+			proc.communicate ()	# output ignored
+		except:
+			logging.critical ("packageManagerEnd.sh failed")
 
 	# delay to leave reboot message up on status lines for a few seconds
 	# and to provide sufficient time for packageManagerEnd.sh to start up

blindInstall/SetupHelperVersion

@@ -1 +1 @@
-v8.0~24
+v8.0~27

changes

@@ -4,16 +4,20 @@ v8.0: (beta)
 	fixed: PackageManager hangs if there is no setup script in package directory
 	fixed: package conflict message disappears
 	fixed: "Once" download scan doesn't check all packages
-	fixed: package checks run unnecessarily until time syncs with NTP
-		after an unclean shutdown
+	fixed: package checks run unnecessarily after an unclean shutdown
+		the real time clock last's value was not updated
+		causing file mod times to be newer than last RTC value
+	fixed: patch fails if pachage was installed by replacement (old install)
 	base script prechecks on file modificaiton times to eliminate unnecessary work
 	setupHelper uninstall and reboots/gui restarts moved from PackageManager main
 		to separate script
 	add TailscaleGX to default package list
 	updatePackage: changed HelperResources version checking to minimize prompts
 	updatePackage: added patch options including MANUAL to prevent automatic patch updates
-	updatePackage: rewrite main file set loop for speed improvement
-
+	updatePackage: rewrite update file sets loop for speed improvement
+	replace OS's patch with one that handles more cases (eg, 0 context) and options
+	fixed: PackageManager hangs on Python 2 (Venus OS prior to v2.80)
+
 v7.18:
 	fixed: only first service is uninstalled
 

reinstallMods

@@ -69,13 +69,13 @@ else
 	logMessage "SetupHelper already installed"
 fi
 
-# tell PackageManager to reinstall the remaining packages"
-logMessage "PackageManager will reinstall remaining packages"
-touch "/etc/venus/REINSTALL_PACKAGES"
-
 # reboot now if signaled from setup script
 if $rebootNeeded ; then
     logMessage "rebooting ..."
     reboot
+else
+	# tell PackageManager to reinstall the remaining packages"
+	logMessage "PackageManager will reinstall remaining packages"
+	touch "/etc/venus/REINSTALL_PACKAGES"
 fi