Merge pull request #163 from MannLabs/develop

Develop

.gitattributes

@@ -0,0 +1,2 @@
+*.html linguist-detectable=false
+*.ipynb linguist-detectable=false

.github/dependabot.yml

@@ -0,0 +1,55 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/requirements"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+  target-branch: develop
+  ignore:
+  - dependency-name: sphinx-rtd-theme
+    versions:
+    - 0.5.2
+  - dependency-name: tqdm
+    versions:
+    - 4.56.2
+    - 4.58.0
+    - 4.60.0
+  - dependency-name: bokeh
+    versions:
+    - 2.3.0
+  - dependency-name: matplotlib
+    versions:
+    - 3.4.0
+    - 3.4.1
+  - dependency-name: ipykernel
+    versions:
+    - 5.5.3
+  - dependency-name: numpy
+    versions:
+    - 1.20.0
+    - 1.20.2
+  - dependency-name: numba
+    versions:
+    - 0.53.1
+  - dependency-name: pyzstd
+    versions:
+    - 0.14.2
+    - 0.14.4
+  - dependency-name: sphinx
+    versions:
+    - 3.5.2
+    - 3.5.3
+  - dependency-name: datashader
+    versions:
+    - 0.12.1
+  - dependency-name: panel
+    versions:
+    - 0.11.1
+  - dependency-name: holoviews
+    versions:
+    - 1.14.2
+  - dependency-name: pandas
+    versions:
+    - 1.2.2
+    - 1.2.3

.github/workflows/cla_assistant.yml

@@ -1,9 +1,10 @@
 name: "CLA Assistant"
 on:
-  issue_comment:
-    types: [created]
-  pull_request:
-    types: [opened,closed,synchronize]
+  # issue_comment:
+  #   types: [created]
+  # pull_request:
+  #   types: [opened,closed,synchronize]
+  workflow_dispatch:
 
 jobs:
   CLAssistant:

README.md

@@ -3,6 +3,7 @@
 [![Downloads](https://pepy.tech/badge/alphatims)](https://pepy.tech/project/alphatims)
 [![Downloads](https://pepy.tech/badge/alphatims/month)](https://pepy.tech/project/alphatims)
 [![Downloads](https://pepy.tech/badge/alphatims/week)](https://pepy.tech/project/alphatims)
+[![Documentation Status](https://readthedocs.org/projects/alphatims/badge/?version=latest)](https://alphatims.readthedocs.io/en/latest/?badge=latest)
 
 ---
 # AlphaTims
@@ -39,8 +40,8 @@ AlphaTims is an open-source Python package that provides fast accession and visu
   * [**Citing AlphaTims**](#citing-alphatims)
   * [**How to contribute**](#how-to-contribute)
   * [**Changelog**](#changelog)
-    * [**0.3.1**](#301)
-    * [**0.3.0**](#300)
+    * [**0.3.1**](#031)
+    * [**0.3.0**](#030)
     * [**0.2.8**](#028)
     * [**0.2.7**](#027)
 
@@ -384,6 +385,16 @@ For more information see [the Contributors License Agreement](misc/CLA.md).
 
 The following changes were introduced in the following versions of AlphaTims. Download the latest version in the [installation section](#installation).
 
+### 0.3.2
+
+  * FEAT: cli/gui allow bruker data as argument.
+  * FEAT/FIX: Polarity included in frame table.
+  * FIX: utils cleanup.
+  * FEAT: cli/gui allow bruker data as argument
+  * FIX: utils issues
+  * FEAT: by default use -1 threads in utils
+  * FIX: disable cla check
+
 ### 0.3.1
 
   * FIX/FEAT: Intensity correction when ICC is used. Note that this is only for exported data, not for visualized data.

alphatims/__init__.py

@@ -2,7 +2,7 @@
 
 
 __project__ = "alphatims"
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 __license__ = "Apache"
 __description__ = "A Python package to index Bruker TimsTOF raw data for fast and easy accession and visualization"
 __author__ = "Sander Willems, Eugenia Voytik"

alphatims/bruker.py

@@ -143,6 +143,7 @@ def read_bruker_sql(
     bruker_d_folder_name: str,
     add_zeroth_frame: bool = True,
     drop_polarity: bool = True,
+    convert_polarity_to_int: bool = True,
 ) -> tuple:
     """Read metadata, (fragment) frames and precursors from a Bruker .d folder.
 
@@ -164,6 +165,11 @@ def read_bruker_sql(
         If False, this column is kept, resulting in a pd.DataFrame with
         dtype=object.
         Default is True.
+    convert_polarity_to_int : bool
+        Convert the polarity to int (-1 or +1).
+        This allows to keep it in numerical form.
+        This is ignored if the polarity is dropped.
+        Default is True.
 
     Returns
     -------
@@ -242,13 +248,21 @@ def read_bruker_sql(
             frames.MaxIntensity[0] = 0
             frames.SummedIntensities[0] = 0
             frames.NumPeaks[0] = 0
+        polarity_col = frames["Polarity"].copy()
         frames = pd.DataFrame(
             {
                 col: pd.to_numeric(
                     frames[col]
                 ) for col in frames if col != "Polarity"
             }
         )
+        if not drop_polarity:
+            if convert_polarity_to_int:
+                frames['Polarity'] = polarity_col.apply(
+                    lambda x: 1 if x == "+" else -1
+                ).astype(np.int8)
+            else:
+                frames['Polarity'] = polarity_col
         return (
             acquisition_mode,
             global_meta_data,
@@ -908,6 +922,8 @@ def __init__(
         use_calibrated_mz_values_as_default: int = 0,
         use_hdf_if_available: bool = False,
         mmap_detector_events: bool = None,
+        drop_polarity: bool = True,
+        convert_polarity_to_int: bool = True,
     ):
         """Create a Bruker TimsTOF object that contains all data in-memory.
 
@@ -953,6 +969,19 @@ def __init__(
             but use an mmap instead. If no .hdf file is available to use for
             mmapping, one will be created automatically.
             Default is False for .d folders and True for .hdf files.
+        drop_polarity : bool
+            The polarity column of the frames table contains "+" or "-" and
+            is not numerical.
+            If True, the polarity column is dropped from the frames table.
+            this ensures a fully numerical pd.DataFrame.
+            If False, this column is kept, resulting in a pd.DataFrame with
+            dtype=object.
+            Default is True.
+        convert_polarity_to_int : bool
+            Convert the polarity to int (-1 or +1).
+            This allows to keep it in numerical form.
+            This is ignored if the polarity is dropped.
+            Default is True.
         """
         logging.info(f"Importing data from {bruker_d_folder_name}")
         if (mmap_detector_events is None) and bruker_d_folder_name.endswith(".hdf"):
@@ -970,29 +999,19 @@ def __init__(
                 self.bruker_d_folder_name = os.path.abspath(
                     bruker_d_folder_name
                 )
-                if mmap_detector_events and hdf_file_exists:
+                if mmap_detector_events:
                     raise IOError(
-                        f"Can only use mmap from .hdf files. "
-                        f"Since {bruker_hdf_file_name} already exists, "
-                        f"and the option use_hdf_if_available=True "
-                        f"is not explicitly used, this would result in "
-                        f"overwriting the .hdf file which could lead to "
-                        f"unexpected data losses. "
-                        f"Either use the .hdf file as input, "
-                        "allow to use the existing .hdf, "
-                        "or remove the existing .hdf file."
+                        f"Can only use mmapping from .hdf files. "
+                        f"Either use the .hdf file as input directly, "
+                        "or use the use_hdf_if_available option."
                     )
                 self._import_data_from_d_folder(
                     bruker_d_folder_name,
                     mz_estimation_from_frame,
                     mobility_estimation_from_frame,
+                    drop_polarity,
+                    convert_polarity_to_int,
                 )
-                if mmap_detector_events:
-                    self._import_data_from_hdf_file(
-                        bruker_d_folder_name,
-                        mmap_detector_events,
-                    )
-                    self.bruker_hdf_file_name = bruker_hdf_file_name
         elif bruker_d_folder_name.endswith(".hdf"):
             self._import_data_from_hdf_file(
                 bruker_d_folder_name,
@@ -1031,6 +1050,8 @@ def _import_data_from_d_folder(
         bruker_d_folder_name: str,
         mz_estimation_from_frame: int,
         mobility_estimation_from_frame: int,
+        drop_polarity: bool = True,
+        convert_polarity_to_int: bool = True,
     ):
         self._version = alphatims.__version__
         self._zeroth_frame = True
@@ -1040,7 +1061,12 @@ def _import_data_from_d_folder(
             self._frames,
             self._fragment_frames,
             self._precursors,
-        ) = read_bruker_sql(bruker_d_folder_name, self._zeroth_frame)
+        ) = read_bruker_sql(
+            bruker_d_folder_name,
+            self._zeroth_frame,
+            drop_polarity,
+            convert_polarity_to_int,
+        )
         self._meta_data = dict(
             zip(global_meta_data.Key, global_meta_data.Value)
         )

alphatims/cli.py

@@ -197,18 +197,22 @@ def cli_option(
                 **parameters,
             )
     else:
+        if "required" in parameters:
+            required = parameters.pop("required")
+        else:
+            required = True
         if "nargs" in parameters:
             return click.argument(
                 parameter_name,
                 type=click.Path(exists=True),
                 nargs=parameters["nargs"],
-                required=True
+                required=required,
             )
         else:
             return click.argument(
                 parameter_name,
                 type=parameters["type"],
-                required=True
+                required=required,
             )
 
 
@@ -231,7 +235,7 @@ def run(ctx, **kwargs):
 
 @run.command("gui", help="Start graphical user interface.")
 @cli_option("port")
-@cli_option("bruker_raw_data", required=False)
+@cli_option("bruker_raw_data", required=False, as_argument=True)
 def gui(port, bruker_raw_data):
     with parse_cli_settings("gui"):
         logging.info("Loading GUI..")

alphatims/utils.py

@@ -14,6 +14,7 @@
 import sys
 import json
 import contextlib
+import multiprocessing
 # local
 import alphatims
 # external
@@ -26,7 +27,7 @@
 LIB_PATH = os.path.join(BASE_PATH, "lib")
 LOG_PATH = os.path.join(BASE_PATH, "logs")
 DOC_PATH = os.path.join(BASE_PATH, "docs")
-MAX_THREADS = 1
+MAX_THREADS = multiprocessing.cpu_count()
 LATEST_GITHUB_INIT_FILE = (
     "https://raw.githubusercontent.com/MannLabs/alphatims/"
     "master/alphatims/__init__.py"
@@ -322,7 +323,6 @@ def set_threads(threads: int, set_global: bool = True) -> int:
     : int
         The number of threads.
     """
-    import multiprocessing
     max_cpu_count = multiprocessing.cpu_count()
     if threads > max_cpu_count:
         threads = max_cpu_count
@@ -505,7 +505,7 @@ def parallel_compiled_func_inner(func):
             cache = True
         numba_func = numba.njit(nogil=True, cache=cache, **kwargs)(func)
 
-        @numba.njit(nogil=True, cache=True)
+        @numba.njit(nogil=True, cache=cache)
         def numba_func_parallel(
             iterable,
             thread_id,
@@ -562,15 +562,16 @@ def wrapper(iterable, *args):
                 threads.append(thread)
             if include_progress_callback:
                 import time
-                progress_count = 0
+                # progress_count = 0
                 progress_bar = 0
+                progress_count = np.sum(progress_counter)
                 for result in progress_callback(
                     iterable,
                     include_progress_callback=include_progress_callback
                 ):
-                    if progress_bar == progress_count:
-                        while progress_count == np.sum(progress_counter):
-                            time.sleep(0.01)
+                    while progress_bar >= progress_count:
+                        time.sleep(0.01)
+                        progress_count = np.sum(progress_counter)
                     progress_bar += 1
             for thread in threads:
                 thread.join()
@@ -759,7 +760,7 @@ def create_hdf_group_from_dict(
                         shuffle=compress,
                         chunks=True if chunked else None,
                     )
-        elif isinstance(value, (bool, int, float, str)):
+        elif isinstance(value, (bool, int, float, str, np.bool_)):
             if overwrite or (key not in hdf_group.attrs):
                 hdf_group.attrs[key] = value
         elif isinstance(value, dict):
@@ -836,21 +837,28 @@ def create_dict_from_hdf_group(
             if (mmap_arrays is not None) and (subgroup.name in mmap_arrays):
                 offset = subgroup.id.get_offset()
                 if offset is not None:
+                    shape = subgroup.shape
                     import mmap
                     with open(parent_file_name, "rb") as raw_hdf_file:
                         mmap_obj = mmap.mmap(
                             raw_hdf_file.fileno(),
                             0,
                             access=mmap.ACCESS_READ
                         )
-                        shape = subgroup.shape
                         result[key] = np.frombuffer(
                             mmap_obj,
                             dtype=subgroup.dtype,
                             count=np.prod(shape),
                             offset=offset
                         ).reshape(shape)
                         # TODO WARNING: mmap is not closed!
+                    # result[key] = np.memmap(
+                    #     dia_data_file_name,
+                    #     dtype=subgroup.dtype,
+                    #     mode="r",
+                    #     offset=offset,
+                    #     shape=shape,
+                    # )
                 else:
                     raise IOError(
                         f"Array {subgroup.name} cannot be mmapped. "

misc/bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.1
+current_version = 0.3.2
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

misc/one_click_linux/control

@@ -1,5 +1,5 @@
 Package: AlphaTims
-Version: 0.3.1
+Version: 0.3.2
 Architecture: all
 Maintainer: Mann Labs <[email protected]>
 Description: AlphaTims GUI

misc/one_click_linux/create_installer_linux.sh

@@ -12,7 +12,7 @@ rm -rf dist
 rm -rf build
 python setup.py sdist bdist_wheel
 cd misc/one_click_linux
-pip install "../../dist/alphatims-0.3.1-py3-none-any.whl[plotting-stable,stable,legacy-stable]"
+pip install "../../dist/alphatims-0.3.2-py3-none-any.whl[plotting-stable,stable,legacy-stable]"
 pip install pyinstaller==4.2
 pyinstaller ../pyinstaller/alphatims.spec -y
 conda deactivate