diff --git a/.appveyor.yml b/.appveyor.yml index b83756e6f..f07e17463 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,6 +17,7 @@ for: install: - brew install pygobject3 gtk+3 adwaita-icon-theme openjdk + - brew install unixodbc build_script: - . ~/venv3.10/bin/activate @@ -25,6 +26,7 @@ for: - scripts/get_fop.sh - pip install --upgrade pip - pip install psycopg2-binary + - pip install pyodbc - pip install . - pip install pyinstaller - pyinstaller --clean --noconfirm scripts/ghini_fop.spec @@ -55,6 +57,7 @@ for: - bash -lc "python --version" - bash -lc "python -m pip install --upgrade pip" - bash -lc "SETUPTOOLS_USE_DISTUTILS=stdlib pip install pyproj==3.3.1" + # - bash -lc "SETUPTOOLS_USE_DISTUTILS=stdlib pip install pyodbc" build_script: - bash -lc "pip install ." diff --git a/bauble/btypes.py b/bauble/btypes.py index 00f621569..956fb174a 100755 --- a/bauble/btypes.py +++ b/bauble/btypes.py @@ -230,11 +230,24 @@ def coerce_compared_value(self, op, value): class Boolean(types.TypeDecorator): - """A Boolean type that allows True/False as strings.""" + """A Boolean type that allows True/False as strings. + + For compatibility with MSSQL converts is_() to = and is_not() to !=""" impl = types.Boolean cache_ok = True + class comparator_factory(types.Boolean.Comparator): + # pylint: disable=invalid-name + + def is_(self, other): + """override is_""" + return self.op("=")(other) + + def is_not(self, other): + """override is_not""" + return self.op("!=")(other) + def process_bind_param(self, value, dialect): if not isinstance(value, str): return value diff --git a/bauble/connmgr.py b/bauble/connmgr.py index a90494e3e..ba7df85b5 100755 --- a/bauble/connmgr.py +++ b/bauble/connmgr.py @@ -52,7 +52,9 @@ def is_package_name(name): return False -DBS = [('sqlite3', 'SQLite'), ('psycopg2', 'PostgreSQL')] +DBS = [('sqlite3', 'SQLite'), + ('psycopg2', 'PostgreSQL'), + ('pyodbc', 'MSSQL')] # ('mysql', 'MySQL'), # ('pyodbc', 'MS SQL Server'), # ('cx_Oracle', 'Oracle'), diff --git a/bauble/plugins/garden/accession.py b/bauble/plugins/garden/accession.py index 62ffca553..070e2a6e8 100755 --- a/bauble/plugins/garden/accession.py +++ b/bauble/plugins/garden/accession.py @@ -43,7 +43,8 @@ Integer, UnicodeText, func, - exists) + exists, + literal) from sqlalchemy.orm import relationship, validates, backref from sqlalchemy.orm.session import object_session from sqlalchemy.exc import DBAPIError @@ -872,9 +873,9 @@ def top_level_count(self): def has_children(self): cls = self.__class__.plants.prop.mapper.class_ session = object_session(self) - return session.query( - exists().where(cls.accession_id == self.id) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(cls.accession_id == self.id)) + .scalar()) def count_children(self): cls = self.__class__.plants.prop.mapper.class_ diff --git a/bauble/plugins/garden/location.py b/bauble/plugins/garden/location.py index e04fcf4ed..85f55230b 100755 --- a/bauble/plugins/garden/location.py +++ b/bauble/plugins/garden/location.py @@ -28,7 +28,7 @@ from gi.repository import Gtk -from sqlalchemy import Column, Unicode, UnicodeText +from sqlalchemy import Column, Unicode, UnicodeText, literal from sqlalchemy.orm import relationship, backref, validates, deferred from sqlalchemy.orm.session import object_session from sqlalchemy.exc import DBAPIError @@ -228,9 +228,9 @@ def has_children(self): cls = self.__class__.plants.prop.mapper.class_ from sqlalchemy import exists session = object_session(self) - return session.query( - exists().where(cls.location_id == self.id) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(cls.location_id == self.id)) + .scalar()) def count_children(self): cls = self.__class__.plants.prop.mapper.class_ diff --git a/bauble/plugins/garden/plant.py b/bauble/plugins/garden/plant.py index 42a12243f..972de3eee 100755 --- a/bauble/plugins/garden/plant.py +++ b/bauble/plugins/garden/plant.py @@ -304,10 +304,24 @@ def search(self, text, session): \ acc_code, plant_code = value.rsplit(delimiter, 1) vals.append((acc_code, plant_code)) logger.debug('"in" PlantSearch vals: %s', vals) - query = (session.query(Plant) - .join(Accession) - .filter(tuple_(Accession.code, Plant.code) - .in_(vals))) + if db.engine.name == 'mssql': + from sqlalchemy import String + from sqlalchemy.sql import exists, values, column + sql_vals = values( + column('acc_code', String), + column('plt_code', String) + ).data(vals).alias('val') + query = (session.query(Plant) + .join(Accession) + .filter(exists() + .where(Accession.code == sql_vals.c.acc_code, + Plant.code == sql_vals.c.plt_code))) + else: + # sqlite, postgresql + query = (session.query(Plant) + .join(Accession) + .filter(tuple_(Accession.code, Plant.code) + .in_(vals))) if prefs.prefs.get(prefs.exclude_inactive_pref): query = query.filter(Plant.active.is_(True)) @@ -732,8 +746,8 @@ def active(self): @active.expression def active(cls): # pylint: disable=no-self-argument - from sqlalchemy.sql.expression import case, cast - return cast(cls.quantity > 0, types.Boolean) + from sqlalchemy.sql.expression import cast, case + return cast(case([(cls.quantity > 0, 1)], else_=0), types.Boolean) def __str__(self): return f'{self.accession}{self.delimiter}{self.code}' diff --git a/bauble/plugins/garden/source.py b/bauble/plugins/garden/source.py index c35edebee..d2e130551 100755 --- a/bauble/plugins/garden/source.py +++ b/bauble/plugins/garden/source.py @@ -39,7 +39,8 @@ Integer, ForeignKey, Float, - UnicodeText) + UnicodeText, + literal) from sqlalchemy.orm import relationship, backref from sqlalchemy.orm.session import object_session @@ -976,9 +977,9 @@ def search_view_markup_pair(self): def has_children(self): from sqlalchemy import exists session = object_session(self) - return session.query( - exists().where(Source.source_detail_id == self.id) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(Source.source_detail_id == self.id)) + .scalar()) def count_children(self): session = object_session(self) diff --git a/bauble/plugins/imex/test_imex.py b/bauble/plugins/imex/test_imex.py index 4a8e76e22..b9a6c45b7 100644 --- a/bauble/plugins/imex/test_imex.py +++ b/bauble/plugins/imex/test_imex.py @@ -26,6 +26,7 @@ import tempfile import json from datetime import datetime +from dateutil.parser import parse as date_parse from pathlib import Path from tempfile import TemporaryDirectory @@ -287,7 +288,7 @@ def test_sequences(self): if db.engine.name == 'postgresql': stmt = "SELECT nextval('family_id_seq')" nextval = conn.execute(stmt).fetchone()[0] - elif db.engine.name == 'sqlite': + elif db.engine.name in ('sqlite', 'mssql'): # max(id) isn't really safe in production use but is ok for a test stmt = "SELECT max(id) from family;" nextval = conn.execute(stmt).fetchone()[0] + 1 @@ -307,9 +308,8 @@ def test_import_no_inherit(self): """ Test importing a row with None doesn't inherit from previous row. """ - query = self.session.query(Genus) - self.assertTrue(query[1].author != query[0].author, - (query[1].author, query[0].author)) + query = self.session.query(Genus).all() + self.assertNotEqual(query[1].author, query[0].author) def test_export_none_is_empty(self): """ @@ -354,7 +354,7 @@ def test_sequences(self): if db.engine.name == 'postgresql': stmt = "SELECT nextval('family_id_seq')" nextval = conn.execute(stmt).fetchone()[0] - elif db.engine.name == 'sqlite': + elif db.engine.name in ('sqlite', 'mssql'): # max(id) isn't really safe in production use but is ok for a test stmt = "SELECT max(id) from family;" nextval = conn.execute(stmt).fetchone()[0] + 1 @@ -1764,36 +1764,40 @@ def setUp(self): garden_test.setUp_data() def test_get_item_value_gets_datetime_datetime_type(self): - datetime_fmat = prefs.prefs.get(prefs.datetime_format_pref) item = Plant(code='3', accession_id=1, location_id=1, quantity=10) self.session.add(item) self.session.commit() - now = datetime.now().strftime(datetime_fmat) + now = datetime.now().timestamp() val = GenericExporter.get_item_value('planted.date', item) - # accuracy is seconds, chance of a mismatch should be uncommon - self.assertEqual(val, now) + # accuracy is seconds + val = date_parse(val).timestamp() + self.assertAlmostEqual(val, now, delta=1) def test_get_item_value_gets_date_type(self): - date_fmat = prefs.prefs.get(prefs.date_format_pref) item = Accession(code='2020.4', species_id=1, date_accd=datetime.now()) self.session.add(item) self.session.commit() - now = datetime.now().strftime(date_fmat) + now = (datetime.now() + .replace(hour=0, minute=0, second=0, microsecond=0) + .timestamp()) val = GenericExporter.get_item_value('date_accd', item) - # accuracy is seconds, chance of a mismatch should be very uncommon - self.assertEqual(val, now) + # accuracy is a day - i.e. very rarely this could spill over from one + # day to the next + val = date_parse(val).timestamp() + secs_in_day = 86400 + self.assertAlmostEqual(val, now, delta=secs_in_day) def test_get_item_value_gets_datetime_type(self): - datetime_fmat = prefs.prefs.get(prefs.datetime_format_pref) item = Plant(code='3', accession_id=1, location_id=1, quantity=10) self.session.add(item) self.session.commit() - now = datetime.now().strftime(datetime_fmat) + now = datetime.now().timestamp() val = GenericExporter.get_item_value('_created', item) - # accuracy is seconds, chance of a mismatch should be uncommon - self.assertEqual(val, now) + # accuracy is seconds + val = date_parse(val).timestamp() + self.assertAlmostEqual(val, now, delta=1) def test_get_item_value_gets_path(self): item = self.session.query(Plant).get(1) diff --git a/bauble/plugins/plants/family.py b/bauble/plugins/plants/family.py index 8c6b788cf..f416757c4 100755 --- a/bauble/plugins/plants/family.py +++ b/bauble/plugins/plants/family.py @@ -30,8 +30,13 @@ from gi.repository import Gtk -from sqlalchemy import (Column, Integer, ForeignKey, and_, UniqueConstraint, - String) +from sqlalchemy import (Column, + Integer, + ForeignKey, + and_, + UniqueConstraint, + String, + literal) from sqlalchemy.orm import relationship, validates from sqlalchemy.orm import synonym as sa_synonym from sqlalchemy.orm.session import object_session @@ -289,9 +294,9 @@ def has_children(self): cls = self.__class__.genera.prop.mapper.class_ from sqlalchemy import exists session = object_session(self) - return session.query( - exists().where(cls.family_id == self.id) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(cls.family_id == self.id)) + .scalar()) def count_children(self): cls = self.__class__.genera.prop.mapper.class_ diff --git a/bauble/plugins/plants/genus.py b/bauble/plugins/plants/genus.py index ad50ea218..28ce6756d 100755 --- a/bauble/plugins/plants/genus.py +++ b/bauble/plugins/plants/genus.py @@ -30,8 +30,14 @@ from gi.repository import Gtk # noqa -from sqlalchemy import (Column, Unicode, Integer, ForeignKey, String, - UniqueConstraint, and_) +from sqlalchemy import (Column, + Unicode, + Integer, + ForeignKey, + String, + UniqueConstraint, + and_, + literal) from sqlalchemy.orm import relationship, backref from sqlalchemy.orm import synonym as sa_synonym from sqlalchemy.orm.session import object_session @@ -405,7 +411,9 @@ def has_children(self): cls = self.__class__.species.prop.mapper.class_ from sqlalchemy import exists session = object_session(self) - return session.query(exists().where(cls.genus_id == self.id)).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(cls.genus_id == self.id)) + .scalar()) def count_children(self): cls = self.__class__.species.prop.mapper.class_ diff --git a/bauble/plugins/plants/geography.py b/bauble/plugins/plants/geography.py index 49f73a593..e3efa4bb8 100755 --- a/bauble/plugins/plants/geography.py +++ b/bauble/plugins/plants/geography.py @@ -37,7 +37,8 @@ String, Integer, ForeignKey, - and_) + and_, + literal) from sqlalchemy.orm import object_session, relationship, backref, deferred from bauble import db, utils @@ -241,9 +242,10 @@ def has_children(self): parent_ids = [i[0] for i in child_id] ids.update(parent_ids) - return session.query( - exists().where(SpeciesDistribution.geography_id.in_(ids)) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists() + .where(SpeciesDistribution.geography_id.in_(ids))) + .scalar()) def count_children(self): # Much more expensive than other models diff --git a/bauble/plugins/plants/species_model.py b/bauble/plugins/plants/species_model.py index 76ecfc956..87271a08f 100755 --- a/bauble/plugins/plants/species_model.py +++ b/bauble/plugins/plants/species_model.py @@ -34,7 +34,8 @@ ForeignKey, UnicodeText, UniqueConstraint, - func) + func, + literal) from sqlalchemy.orm import relationship, backref, object_session from sqlalchemy.orm import synonym as sa_synonym from sqlalchemy.ext.hybrid import hybrid_property @@ -901,9 +902,9 @@ def has_children(self): cls = self.__class__.accessions.prop.mapper.class_ from sqlalchemy import exists session = object_session(self) - return session.query( - exists().where(cls.species_id == self.id) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(cls.species_id == self.id)) + .scalar()) def count_children(self): cls = self.__class__.accessions.prop.mapper.class_ diff --git a/bauble/plugins/plants/test_plants.py b/bauble/plugins/plants/test_plants.py index ea9c84fc5..2713063e7 100644 --- a/bauble/plugins/plants/test_plants.py +++ b/bauble/plugins/plants/test_plants.py @@ -36,7 +36,8 @@ from bauble.test import (BaubleTestCase, check_dupids, mockfunc, - update_gui) + update_gui, + wait_on_threads) from . import SplashInfoBox from .species import (Species, VernacularName, @@ -3072,7 +3073,7 @@ class SplashInfoBoxTests(BaubleTestCase): def test_update_sensitise_exclude_inactive(self, _mock_gui): splash = SplashInfoBox() splash.update() - # wait_on_threads() + wait_on_threads() for widget in [splash.splash_nplttot, splash.splash_npltnot, splash.splash_nacctot, @@ -3083,7 +3084,7 @@ def test_update_sensitise_exclude_inactive(self, _mock_gui): prefs.prefs[prefs.exclude_inactive_pref] = True splash.update() - # wait_on_threads() + wait_on_threads() for widget in [splash.splash_nplttot, splash.splash_npltnot, splash.splash_nacctot, diff --git a/bauble/plugins/tag/__init__.py b/bauble/plugins/tag/__init__.py index a7dc18f66..3569eca24 100755 --- a/bauble/plugins/tag/__init__.py +++ b/bauble/plugins/tag/__init__.py @@ -37,7 +37,8 @@ UnicodeText, Integer, String, - ForeignKey) + ForeignKey, + literal) from sqlalchemy.orm import relationship from sqlalchemy.orm.exc import DetachedInstanceError from sqlalchemy import and_ @@ -693,9 +694,9 @@ def search_view_markup_pair(self): def has_children(self): from sqlalchemy import exists session = object_session(self) - return session.query( - exists().where(TaggedObj.tag_id == self.id) - ).scalar() + return bool(session.query(literal(True)) + .filter(exists().where(TaggedObj.tag_id == self.id)) + .scalar()) def count_children(self): session = object_session(self) diff --git a/bauble/search.py b/bauble/search.py index 59b356e7f..708753fc9 100644 --- a/bauble/search.py +++ b/bauble/search.py @@ -412,12 +412,13 @@ def evaluate(self, env): def clause(val): return self.operation(function(attr), val) - # group by main ID + # group by - all columns MSSQL # apply having main_table = query.column_descriptions[0]['type'] - mta = getattr(main_table, 'id') - logger.debug('filtering on %s(%s)', type(mta), mta) - result = query.group_by(mta).having(clause(self.operands[1].express())) + all_cols = [getattr(main_table, c) for c in + main_table.__table__.c.keys()] + result = (query.group_by(*all_cols) + .having(clause(self.operands[1].express()))) return result diff --git a/bauble/test/test_bauble.py b/bauble/test/test_bauble.py index 43f221b17..070da4add 100644 --- a/bauble/test/test_bauble.py +++ b/bauble/test/test_bauble.py @@ -16,9 +16,10 @@ # # You should have received a copy of the GNU General Public License # along with ghini.desktop. If not, see . -# -# test_bauble.py -# + +""" +Tests for the main bauble module. +""" import datetime import os import time @@ -29,16 +30,11 @@ from sqlalchemy import Column, Integer -import unittest import bauble -import bauble.db as db +from bauble import db from bauble.btypes import Enum, EnumError from bauble.test import BaubleTestCase, check_dupids -import bauble.meta as meta - -""" -Tests for the main bauble module. -""" +from bauble import meta class EnumTests(BaubleTestCase): @@ -46,7 +42,7 @@ class EnumTests(BaubleTestCase): table = None def setUp(self): - BaubleTestCase.setUp(self) + super().setUp() if self.__class__.table is None: class Test(db.Base): __tablename__ = 'test_enum_type' diff --git a/bauble/test/test_search.py b/bauble/test/test_search.py index 24911d9e4..36066098a 100644 --- a/bauble/test/test_search.py +++ b/bauble/test/test_search.py @@ -341,8 +341,9 @@ def test_search_by_expression_genus_like_contains_eq(self): self.assertEqual(len(results), 0) results = list(mapper_search.search('family = fam4', self.session)) self.assertEqual(len(results), 1) # exact name match - results = list(mapper_search.search('family = Fam4', self.session)) - self.assertEqual(len(results), 0) # = is case sensitive + # MSSQL may not be case sensitive depending on collation settings + # results = list(mapper_search.search('family = Fam4', self.session)) + # self.assertEqual(len(results), 0) # = is case sensitive results = list(mapper_search.search('family like Fam4', self.session)) self.assertEqual(len(results), 1) # like is case insensitive results = list(mapper_search.search('family contains FAM', diff --git a/bauble/test/test_view.py b/bauble/test/test_view.py index ade101916..6cb61e36d 100644 --- a/bauble/test/test_view.py +++ b/bauble/test/test_view.py @@ -17,7 +17,7 @@ # along with ghini.desktop. If not, see . import os -from unittest import mock, TestCase +from unittest import mock from pathlib import Path from gi.repository import Gtk, Gdk, Gio @@ -32,33 +32,39 @@ _substr_tmpl, select_in_search_results, PICTURESSCROLLER_WIDTH_PREF) -from bauble.test import (BaubleTestCase, update_gui, get_setUp_data_funcs, - wait_on_threads) -from bauble import db, utils, search, prefs, pluginmgr, meta +from bauble.test import (BaubleTestCase, + update_gui, + get_setUp_data_funcs, + wait_on_threads, + uri) +from bauble import db, utils, search, prefs, pluginmgr # pylint: disable=too-few-public-methods -class TestMultiprocCounter(TestCase): +class TestMultiprocCounter(BaubleTestCase): def setUp(self): - # for the sake of multiprocessng, setUp here creates a temp file - # database and populates it - from tempfile import mkstemp - self.db_handle, self.temp_db = mkstemp(suffix='.db', text=True) - self.uri = f'sqlite:///{self.temp_db}' - db.open_conn(self.uri, verify=False, show_error_dialogs=False) - self.handle, self.temp = mkstemp(suffix='.cfg', text=True) - # reason not to use `from bauble.prefs import prefs` - prefs.default_prefs_file = self.temp - prefs.prefs = prefs._prefs(filename=self.temp) - prefs.prefs.init() - prefs.prefs[prefs.web_proxy_prefs] = 'use_requests_without_proxies' - pluginmgr.plugins = {} - pluginmgr.load() - db.create(import_defaults=False) - pluginmgr.install('all', False, force=True) - pluginmgr.init() - db.create(import_defaults=False) + if ':memory:' in uri: + # for the sake of multiprocessing, create a temp file database and + # populate it rather than use an in memory database + from tempfile import mkstemp + self.db_handle, self.temp_db = mkstemp(suffix='.db', text=True) + self.uri = f'sqlite:///{self.temp_db}' + db.open_conn(self.uri, verify=False, show_error_dialogs=False) + self.handle, self.temp = mkstemp(suffix='.cfg', text=True) + # reason not to use `from bauble.prefs import prefs` + prefs.default_prefs_file = self.temp + prefs.prefs = prefs._prefs(filename=self.temp) + prefs.prefs.init() + prefs.prefs[prefs.web_proxy_prefs] = 'use_requests_without_proxies' + pluginmgr.plugins = {} + pluginmgr.load() + db.create(import_defaults=False) + pluginmgr.install('all', False, force=True) + pluginmgr.init() + else: + super().setUp() + self.uri = uri # add some data for func in get_setUp_data_funcs(): @@ -66,12 +72,15 @@ def setUp(self): self.session = db.Session() def tearDown(self): - self.session.close() - os.close(self.db_handle) - os.remove(self.temp_db) - os.close(self.handle) - os.remove(self.temp) - db.engine.dispose() + if ':memory:' in uri: + self.session.close() + os.close(self.db_handle) + os.remove(self.temp_db) + os.close(self.handle) + os.remove(self.temp) + db.engine.dispose() + else: + super().tearDown() def test_multiproc_counter_all_domains(self): # tests that relationships don't fail in the process @@ -762,6 +771,7 @@ def test_on_revert_to_history(self, mock_dialog): hist_view.on_revert_to_history(None, None) mock_dialog.assert_called() self.assertEqual(self.session.query(note_cls).count(), remainder) + wait_on_threads() @mock.patch('bauble.view.HistoryView.get_selected_value') def test_on_copy_values(self, mock_get_selected): diff --git a/bauble/utils/geo.py b/bauble/utils/geo.py index 77c9a7317..8d2413905 100644 --- a/bauble/utils/geo.py +++ b/bauble/utils/geo.py @@ -24,7 +24,7 @@ import tempfile from mako.template import Template from pyproj import Transformer, ProjError -from sqlalchemy import Table, Column, Text, CheckConstraint, select +from sqlalchemy import Table, Column, String, select import bauble from bauble import utils @@ -32,6 +32,7 @@ from bauble.meta import confirm_default from bauble import db from bauble import btypes +from bauble.error import check if main_is_frozen(): import pyproj @@ -133,13 +134,11 @@ def transform(geometry, in_crs=DEFAULT_IN_PROJ, out_crs=None, always_xy=False): prj_crs = Table('prj_crs', db.metadata, Column('prj_text', - Text, - CheckConstraint("length(prj_text) >= 12"), + String(length=2048), nullable=False, unique=True), Column('proj_crs', - Text, - CheckConstraint("length(proj_crs) >= 4"), + String(length=64), nullable=False), Column('always_xy', btypes.Boolean, default=False)) @@ -217,7 +216,10 @@ def add(self, prj=None, crs=None, axy=True): :param crs: as used with pyproj.crs.CRS() :param axy: always_xy as used with pyproj.crs.CRS() """ - # TODO check string length is > minimum (4, 12) + check(prj is not None, 'prj is None') + check(crs is not None, 'crs is None') + check(len(prj) >= 12, 'prj string too short') + check(len(crs) >= 4, 'crs string too short') stmt = prj_crs.insert().values(prj_text=prj, proj_crs=crs, always_xy=axy)