diff --git a/.gitignore b/.gitignore index 3563ff1..31c0894 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ myTree.html myTree.kml .vs/ other/ +geodat-address-cache-1.csv +geodat-address-cache-2.csv +*.user # Byte-compiled / optimized / DLL files @@ -135,3 +138,10 @@ dmypy.json # Pyre type checker .pyre/ +/gedcom-to-map/webdrive.py +/samples/sample-bourbon +/samples/sample-kennedy +/samples/shakespeare.html +/Tree.html +/Tree.kml +*.pyperf diff --git a/README.md b/README.md index 5e03dec..55add0a 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,15 @@ python3 ..\gedcom-to-map\gedcom-to-map.py pres2020.ged pres2020-2 -main "@I676@" ## Running on Linux - [See Running on WSL](docs/running-on-wsl.md) +## Other Ideas +- [See Exploring Family trees](docs/otherlearnings.md) + +## Comparing MyHeritage PedigreeMap Heatmap and GedcomVisual Heatmap +I noticed that the MyHeritage has added a heatmap a year or so ago and it has a lot of overlap with the GedcomVisual heatmap. + +![img](docs/MyHeritage-2023-10-09.png) and ![img](docs/GedcomVisual-2023-10-09.png) + + ## TODO - Add a treed hierarchy selector to enable people as groups and add expand/collapse to navigation - more troubleshooting on the address lookup @@ -172,21 +181,27 @@ python3 ..\gedcom-to-map\gedcom-to-map.py pres2020.ged pres2020-2 -main "@I676@" - create a marker animation by year (in time steps) - better save the state of the options for next use - in Person dialog show something for people still alive (vs None or Unknown) -- reduce loop cycle on idle -- better error checking for badly formed GED files +- remember the starting person next time the same GED is opened ## Issues - Marriage is not read correctly all the time, does not deal with multiple marriages + ### GUI - Need to separate the Load and GPS resolve steps - Need to better detect change to the GPS cache file +- need better interaction with buttons and menu selections - could be memory leak issues - need to determine how do deal with very large HTML files - logging panel leaks over the people grid - ## Releases +### v0.2.3.5 +- Adjusted Sorting for People list, adjusted adddress and year value for unknown. +- Try and fix the busy icon +- Error catching for parsing input of Gedcom files +- fixed - Uses the correct cache file from the Selected input GED directory +- file selection dialog automatically opens file ### v0.2.3.4 - Added dynamic highlighting based on main selection for HTML - Added Statistics menu under Actions diff --git a/docs/MyHeritage-2023-10-09.png b/docs/MyHeritage-2023-10-09.png new file mode 100644 index 0000000..e9b5c7e Binary files /dev/null and b/docs/MyHeritage-2023-10-09.png differ diff --git a/docs/gedcomVisual-2023-10-09.png b/docs/gedcomVisual-2023-10-09.png new file mode 100644 index 0000000..a09463b Binary files /dev/null and b/docs/gedcomVisual-2023-10-09.png differ diff --git a/gedcom-to-map/const.py b/gedcom-to-map/const.py index 9f2b990..c6e1245 100644 --- a/gedcom-to-map/const.py +++ b/gedcom-to-map/const.py @@ -1,7 +1,8 @@ """Constants for gedcom-to-visualmap""" -VERSION = "0.2.3.4" +VERSION = "0.2.3.5" NAME = "gedcom-to-visualmap" +GUINAME = 'GEDCOM Visual Map' GV_COUNTRIES_JSON = 'https://raw.githubusercontent.com/nnjeim/world/master/resources/json/countries.json' GV_STATES_JSON = 'https://raw.githubusercontent.com/nnjeim/world/master/resources/json/states.json' diff --git a/gedcom-to-map/gedcom/gpslookup.py b/gedcom-to-map/gedcom/gpslookup.py index de81411..040eb60 100644 --- a/gedcom-to-map/gedcom/gpslookup.py +++ b/gedcom-to-map/gedcom/gpslookup.py @@ -24,14 +24,18 @@ fixuplist = {r'\bof\s': '', r'\spart of\s' : ' ', r'po box [0-9]+\s' : ' ', + r'\([\w\s\.]+\)' : '', '(town)' : '', '(town/ville)' : '', + '(west/ouest)' : '', 'Upper Canada, British Colonial America': 'Ontario, Canada', 'Lower Canada, British Colonial America': 'Quebec, Canada', 'British Colonial America': 'Canada'} wordfixlist = {'of': '', r'co\.' : 'county', - r'twp\.': 'township'} + r'twp\.': 'township', + r'tp\.': 'township', + r'tp': 'township'} geoapp = None cache_filename = (r"geodat-address-cache.csv", r"geodat-address-cache-1.csv", r"geodat-address-cache-2.csv") @@ -41,7 +45,7 @@ defaultcountry = "CA" - +# The cache files should not change, but they might in the future. There should be an option to clear the cache files. #TODO def readCachedURL(cfile, url): nfile = os.path.join(tempfile.gettempdir() , cfile) if not os.path.exists(nfile): @@ -93,6 +97,10 @@ def __init__(self, humans, gvO: gvOptions): self.humans = humans self.countrieslist = None self.Geoapp = None + self.stats = "" + self.used = 0 + self.usedNone = 0 + self.totaladdr = 0 # This pulls from the same directory as GEDCOM file if gvO.resultpath: @@ -175,6 +183,23 @@ def __init__(self, humans, gvO: gvOptions): self.states[(self.stateslist[c]['name']).lower()] = self.stateslist[c] self.wordxlat = WordXlator(wordfixlist) self.xlat = Xlator(fixuplist) + self.updatestats() + + + def updatestats(self): + self.used = 0 + self.usedNone = 0 + self.totaladdr = 0 + if hasattr(self, 'addresses') and self.addresses: + for xaddr in self.addresses.keys(): + + if (self.addresses[xaddr]['used'] > 0): + self.used += 1 + self.totaladdr += self.addresses[xaddr]['used'] + if (self.addresses[xaddr]['lat'] is None): self.usedNone += 1 + + self.stats = f"Unique addresses: {self.used} with unresolvable: {self.usedNone}" + def saveAddressCache(self): if self.usecacheonly or self.gOptions.ShouldStop(): @@ -191,9 +216,6 @@ def saveAddressCache(self): return else: logger.warning("No Addresses in addresslist") - used = 0 - usedNone = 0 - totaladdr = 0 n = ['','',''] if self.addresses: resultpath = self.gOptions.resultpath @@ -251,15 +273,12 @@ def saveAddressCache(self): ] csvwriter.writerow(r) - for xaddr in self.addresses.keys(): - if (self.addresses[xaddr]['used'] > 0): - used += 1 - totaladdr += self.addresses[xaddr]['used'] - if (self.addresses[xaddr]['lat'] is None): usedNone += 1 - - logger.info("Unique addresses: %d with %d have missing GPS for a Total of %d",used, usedNone, totaladdr) - self.gOptions.step(f"Cache Table is {totaladdr} addresses") + + self.updatestats() + logger.info("Unique addresses: %d with %d have missing GPS for a Total of %d",self.used, self.usedNone, self.totaladdr) + + self.gOptions.step(f"Cache Table is {self.totaladdr} addresses") def improveaddress(self,theaddress, thecountry= None): @@ -477,5 +496,10 @@ def resolveaddresses(self, humans): if (humans[human].death and humans[human].death.where): humans[human].death.pos = self.lookupaddresses(humans[human].death.where) logger.debug ("{:30} @ D {:60} = {}".format(humans[human].name, humans[human].death.where, humans[human].death.pos )) + self.updatestats() + + + + diff --git a/gedcom-to-map/gedcomVisualGUI.py b/gedcom-to-map/gedcomVisualGUI.py index ad112a6..2919503 100644 --- a/gedcom-to-map/gedcomVisualGUI.py +++ b/gedcom-to-map/gedcomVisualGUI.py @@ -14,6 +14,7 @@ import sys import os.path import time +from warnings import catch_warnings import wx # pylint: disable=no-member import wx.lib.anchors as anchors @@ -24,7 +25,7 @@ import logging import logging.config -from const import NAME, VERSION, LOG_CONFIG, KMLMAPSURL, GVFONTSIZE, GVFONT +from const import NAME, VERSION, GUINAME, LOG_CONFIG, KMLMAPSURL, GVFONTSIZE, GVFONT from gedcomoptions import gvOptions from gedcomvisual import doKML, doHTML, ParseAndGPS @@ -240,6 +241,8 @@ def OnFileOpenDialog(self, evt): self.filehistory.Save(panel.config) dlg.Destroy() + wx.Yield() + panel.LoadGEDCOM() def Cleanup(self, *args): # A little extra cleanup is required for the FileHistory control @@ -256,6 +259,8 @@ def OnFileHistory(self, evt): # add it back to the history so it will be moved up the list self.filehistory.AddFileToHistory(path) panel.setInputFile(path) + wx.Yield() + panel.LoadGEDCOM() def OnAbout(self, event): """Display an About Dialog""" @@ -264,14 +269,21 @@ def OnAbout(self, event): wx.OK|wx.ICON_INFORMATION) def OnInfo(self, event): """Display an Staticis Info Dialog""" + + withoutaddr = 0 msg = "" if hasattr(panel.gO, 'humans') and panel.gO.humans: + # for xh in panel.gO.humans.keys(): + # if (panel.gO.humans[xh].bestlocation() == ''): + # withoutaddr += 1 + # msg = f'Total People :\t{len(panel.gO.humans)}\n People without any address {withoutaddr}' msg = f'Total People :\t{len(panel.gO.humans)}' else: msg = "No people loaded yet" msg = msg + '\n' if hasattr(panel.gO, 'lookup') and hasattr(panel.gO.lookup, 'addresses') and panel.gO.lookup.addresses: - msg = msg + f'Total addresses :\t{len(panel.gO.lookup.addresses)}' + msg = msg + f'Total addresses :\t{len(panel.gO.lookup.addresses)}' + '\n' + panel.gO.lookup.stats + else: msg = msg + "No address in cache" @@ -438,7 +450,7 @@ def PopulateList(self, humans, mainperson): self.list.SetColumnWidth(1, wx.LIST_AUTOSIZE_USEHEADER) self.list.SetColumnWidth(2, wx.LIST_AUTOSIZE_USEHEADER) self.list.SetColumnWidth(3, wx.LIST_AUTOSIZE_USEHEADER) - self.list.SetColumnWidth(4, 60) + self.list.SetColumnWidth(4, wx.LIST_AUTOSIZE_USEHEADER) # show how to select an item @@ -990,10 +1002,12 @@ def OnMyTimer(self, evt): def OnBusyStart(self, evt): self.d.ai.Start() self.d.ai.Show() + wx.Yield() def OnBusyStop(self, evt): self.d.ai.Stop() self.d.ai.Hide() + wx.Yield() def OnUpdate(self, evt): # proces evt state hand off @@ -1110,8 +1124,22 @@ def updateOptions(self): pass def LoadGEDCOM(self): - self.OnBusyStart(-1) - self.threads[0].Trigger(1) + #TODO stop the previous actions and then do the load... need to be improved + if self.threads[0].IsTriggered(): + self.gO.stopping = True + else: + self.OnBusyStart(-1) + time.sleep(0.1) + + cachepath, _ = os.path.split(self.gO.get('GEDCOMinput')) + if self.gO.get('gpsfile'): + sourcepath, _ = os.path.split(self.gO.get('gpsfile')) + else: + sourcepath = None + if self.gO.lookup and cachepath != sourcepath: + del self.gO.lookup + self.gO.lookup = None + self.threads[0].Trigger(1) def DrawGEDCOM(self): @@ -1281,9 +1309,13 @@ def Start(self): def Stop(self): self.keepGoing = False + def IsRunning(self): return self.running + def IsTriggered(self): + return self.do != 0 + def Trigger(self, dolevel): if dolevel & 1 or dolevel & 4: panel.d.BTNLoad.SetBackgroundColour(panel.d.CLR_BTN_DONE) @@ -1304,21 +1336,37 @@ def Run(self): if self.do != 0: logger.info("triggered thread %d", self.do) self.gOptions.stopping = False + wx.Yield() if self.do & 1 or (self.do & 4 and not self.gOptions.parsed): + evt = UpdateBackgroundEvent(value='busy') + wx.PostEvent(self.win, evt) + wx.Yield() logger.info("start ParseAndGPS") if hasattr(self, 'humans'): if self.humans: del self.humans self.gOptions.humans = None + self.humans = None logger.info("ParseAndGPS") - self.humans = ParseAndGPS(self.gOptions, 1) - if self.humans: - self.updategrid = True - evt = UpdateBackgroundEvent(value='busy') - wx.PostEvent(self.win, evt) - time.sleep(0.25) + try: + self.humans = ParseAndGPS(self.gOptions, 1) - + except Exception as e: + # Capture other exceptions + if hasattr(self, 'humans'): + if self.humans: + del self.humans + self.humans = None + self.do = 0 + self.gOptions.stopping = False + self.AddInfo(str(e), True) + logger.info(str(e)) + + self.updategrid = True + evt = UpdateBackgroundEvent(value='busy') + wx.PostEvent(self.win, evt) + wx.Yield() + if hasattr (self, 'humans') and self.humans: logger.info("human count %d", len(self.humans)) self.humans = ParseAndGPS(self.gOptions, 2) self.updategrid = True @@ -1327,7 +1375,7 @@ def Run(self): if self.gOptions.Main: self.AddInfo(f" with '{self.gOptions.Main}' as starting person", False) else: - self.AddInfo("File could not be read as a GEDCOM file", False) + self.AddInfo("File could not be read as a GEDCOM file", True) if self.do & 2: logger.info("start do 2") @@ -1370,7 +1418,7 @@ def Run(self): logger.setLevel(logging.DEBUG) logger.info("Starting up %s %s", NAME, VERSION) app = wx.App() - frm = VisualMapFrame(None, title='GEDCOM Visual Map', size=(800, 800), style = wx.DEFAULT_FRAME_STYLE) + frm = VisualMapFrame(None, title=GUINAME, size=(1024, 800), style = wx.DEFAULT_FRAME_STYLE) panel = VisualMapPanel(frm) panel.SetupOptions() frm.filehistory.Load(panel.config) diff --git a/gedcom-to-map/models/Human.py b/gedcom-to-map/models/Human.py index 36fd48d..afc80c0 100644 --- a/gedcom-to-map/models/Human.py +++ b/gedcom-to-map/models/Human.py @@ -33,22 +33,22 @@ def __repr__(self): def refyear(self): - bestyear = "Unknown" + bestyear = "?Unknown" if self.birth and self.birth.when: year = self.birth.whenyear() - bestyear = "Born " + self.birth.whenyear() if year else "Unknown" + bestyear = self.birth.whenyear() + " (Born)" if year else "?Unknown" elif self.death and self.death.when: year = self.death.whenyear() - bestyear = "Died " + self.death.whenyear() if year else "Unknown" + bestyear = self.death.whenyear() + " (Died)" if year else "?Unknown" return bestyear def bestlocation(self): # TODO Best Location should consider if in KML mode and what is selected best = ["Unknown", ""] if self.birth and self.birth.pos: - best = [str(self.birth.pos), "Born " + self.birth.where if self.birth.where else "Unknown"] + best = [str(self.birth.pos), self.birth.where + " (Born)" if self.birth.where else ""] elif self.death and self.death.pos: - best = [str(self.death.pos), "Died " + self.death.where if self.death.where else "Unknown"] + best = [str(self.death.pos), self.death.where + " (Died)" if self.death.where else ""] return best diff --git a/gedcom-to-map/render/foliumExp.py b/gedcom-to-map/render/foliumExp.py index 2955504..4a557d9 100644 --- a/gedcom-to-map/render/foliumExp.py +++ b/gedcom-to-map/render/foliumExp.py @@ -294,8 +294,8 @@ def export(self, main: Pos, lines: [Line], saveresult = True): gn = lgd_txt.format( txt=labelname, col= lc) fm_line = [] - bextra = "Born {}".format(line.human.birth.whenyear()) if line.human.birth and line.human.birth.when else '' - dextra = "Died {}".format(line.human.death.whenyear()) if line.human.death and line.human.death.when else '' + bextra = "{} (Born)".format(line.human.birth.whenyear()) if line.human.birth and line.human.birth.when else '' + dextra = "{} (Died)".format(line.human.death.whenyear()) if line.human.death and line.human.death.when else '' fancyname = fancyname + "
" + bextra +" "+ dextra if (bextra != '') or (dextra != '') else fancyname fancypopup = f"
{fancyname}
" if line.human.photo: