   1# -*- coding: utf-8 -*-
      2"""
      3Created on Wed Apr 12 16:47:19 2023
      4
      5@author: Bart Steemans. Govers Lab.
      6"""
      7
      8import os
      9import pickle
     10import cv2 as cv
     11import numpy as np
     12import tifffile
     13from . import utilities as u
     14from .logging_utils import get_bactoscoop_logger, tqdm_if_verbose
     15from .image import Image
     16import pandas as pd
     17from itertools import combinations
     18
     19import logging, sys, datetime, traceback
     20import gc
     21import tempfile
     22
     23
     24bactoscoop_logger = get_bactoscoop_logger()
     25# # Change following to DEBUG to see more information
     26# bactoscoop_logger.setLevel(logging.INFO)
     27
     28
     29class ImageCollection:
     30    """ """
     31
     32    def __init__(
     33        self, image_folder_path=None, px=0.065, error_policy="continue",
     34        log_cell_errors=False,
     35    ):
     36        """Create a collection.
     37
     38        ``error_policy="continue"`` records item-level errors and proceeds with
     39        independent fields. ``error_policy="raise"`` raises on the first one.
     40        Required-stage failures still stop dependent work in either mode.
     41        Cell feature failures are recorded separately and do not make a run
     42        partial in continue mode. Set ``log_cell_errors=True`` to print each one.
     43        """
     44        if error_policy not in {"continue", "raise"}:
     45            raise ValueError("error_policy must be either 'continue' or 'raise'.")
     46        self.error_policy = error_policy
     47        self.log_cell_errors = log_cell_errors
     48        self.processing_errors = []
     49        self.cell_errors = []
     50        self.completed_stages = set()
     51
     52        self.px = px  # microns per pixel
     53        self.images = None
     54        self.masks = None
     55        self.channel_images = {}
     56        self.channel_image_filenames = {}
     57        self.channel_images_by_field = {}
     58        self.inverted_image = None
     59        self.channel_list = None
     60        self.image_filenames = None
     61        self.mask_filenames = None
     62        self.phase_channel = ""
     63        self.segmentation_failed = False
     64
     65        self.image_folder_path = image_folder_path  # Path to the folder to be processed
     66        self.name = (
     67            os.path.basename(os.path.normpath(self.image_folder_path))
     68            if self.image_folder_path is not None
     69            else None
     70        )
     71
     72        self.image_objects = []  # Array that will be populated with objects
     73
     74        self.mesh_df_collection = pd.DataFrame()
     75
     76        self.feature_dataframes = {}
     77        self.merged_features = None
     78        self.mesh_prefilter_stats = pd.DataFrame()
     79
     80    def _require_image_folder_path(self):
     81        if self.image_folder_path is None:
     82            raise ValueError("image_folder_path must be set before loading image data.")
     83
     84    def _require_loaded_images(self):
     85        if not self.image_objects:
     86            raise ValueError(
     87                "No image objects are available. Run create_image_objects() or load mesh data first."
     88            )
     89
     90    def _record_processing_error(
     91        self, stage, error, image_name=None, channel=None, cell_id=None
     92    ):
     93        record = {
     94            "stage": stage,
     95            "image_name": image_name,
     96            "channel": channel,
     97            "cell_id": cell_id,
     98            "error_type": type(error).__name__,
     99            "error": str(error),
    100        }
    101        self.processing_errors.append(record)
    102        bactoscoop_logger.error(
    103            "Processing error at %s (image=%s, channel=%s, cell_id=%s): %s",
    104            stage,
    105            image_name,
    106            channel,
    107            cell_id,
    108            error,
    109        )
    110        if self.error_policy == "raise":
    111            raise error
    112        return record
    113
    114    def _record_cell_error(
    115        self, stage, error, image_name, channel, cell_id, error_type=None
    116    ):
    117        record = {
    118            "stage": stage,
    119            "image_name": image_name,
    120            "channel": channel,
    121            "cell_id": cell_id,
    122            "error_type": error_type or type(error).__name__,
    123            "error": str(error),
    124        }
    125        self.cell_errors.append(record)
    126        if self.log_cell_errors:
    127            bactoscoop_logger.warning(
    128                "Cell processing error at %s (image=%s, channel=%s, cell_id=%s): %s",
    129                stage, image_name, channel, cell_id, error,
    130            )
    131        if self.error_policy == "raise":
    132            raise error
    133        return record
    134
    135    def processing_summary(self):
    136        """Return attempted-stage status and retained image/cell error records."""
    137        if self.processing_errors:
    138            status = "completed_with_errors"
    139        elif self.completed_stages:
    140            status = "completed"
    141        else:
    142            status = "not_started"
    143        return {
    144            "status": status,
    145            "completed_stages": sorted(self.completed_stages),
    146            "error_count": len(self.processing_errors),
    147            "errors": list(self.processing_errors),
    148            "cell_error_count": len(self.cell_errors),
    149            "cell_errors": list(self.cell_errors),
    150        }
    151
    152    def _record_stage_success(self, stage):
    153        self.completed_stages.add(stage)
    154
    155    def _validate_phase_image_state(self):
    156        if self.images is None or self.image_filenames is None:
    157            raise ValueError(
    158                "Phase images are not loaded. Run load_phase_images() before creating image objects."
    159            )
    160        if self.masks is None:
    161            raise ValueError(
    162                "Masks are not loaded. Run load_masks() before creating image objects."
    163            )
    164        if len(self.image_filenames) != len(self.images):
    165            raise ValueError(
    166                f"Loaded {len(self.image_filenames)} image names but {len(self.images)} phase images."
    167            )
    168        if self.mask_filenames is None or len(self.mask_filenames) != len(self.masks):
    169            raise ValueError("Mask arrays and mask filenames are not aligned.")
    170
    171        phase_by_key = {}
    172        for index, filename in enumerate(self.image_filenames):
    173            key = u.image_field_key(filename, suffix=self.phase_channel)
    174            if key in phase_by_key:
    175                self._record_processing_error(
    176                    "phase_mask_matching",
    177                    ValueError(f"Duplicate phase field identity '{key}'."),
    178                    image_name=filename,
    179                )
    180                continue
    181            phase_by_key[key] = (index, filename, self.images[index])
    182
    183        masks_by_key = {}
    184        for index, filename in enumerate(self.mask_filenames):
    185            key = u.image_field_key(
    186                filename, suffix=self.phase_channel, is_mask=True
    187            )
    188            if key in masks_by_key:
    189                self._record_processing_error(
    190                    "phase_mask_matching",
    191                    ValueError(f"Duplicate mask field identity '{key}'."),
    192                    image_name=filename,
    193                )
    194                continue
    195            masks_by_key[key] = (filename, self.masks[index])
    196
    197        matched = []
    198        for key, (frame, image_name, image) in phase_by_key.items():
    199            mask_entry = masks_by_key.get(key)
    200            if mask_entry is None:
    201                self._record_processing_error(
    202                    "phase_mask_matching",
    203                    ValueError(f"No mask matches phase field '{image_name}'."),
    204                    image_name=image_name,
    205                )
    206                continue
    207            mask_name, mask = mask_entry
    208            if image.ndim != 2 or mask.ndim != 2:
    209                self._record_processing_error(
    210                    "phase_mask_matching",
    211                    ValueError(
    212                        f"Phase and mask images must be 2D; got phase shape "
    213                        f"{image.shape} and mask '{mask_name}' shape {mask.shape}."
    214                    ),
    215                    image_name=image_name,
    216                )
    217                continue
    218            if image.shape != mask.shape:
    219                self._record_processing_error(
    220                    "phase_mask_matching",
    221                    ValueError(
    222                        f"Phase shape {image.shape} does not match mask "
    223                        f"'{mask_name}' shape {mask.shape}."
    224                    ),
    225                    image_name=image_name,
    226                )
    227                continue
    228            matched.append((frame, image_name, image, mask))
    229
    230        for key, (mask_name, _) in masks_by_key.items():
    231            if key not in phase_by_key:
    232                self._record_processing_error(
    233                    "phase_mask_matching",
    234                    ValueError(f"No phase image matches mask '{mask_name}'."),
    235                    image_name=mask_name,
    236                )
    237        if not matched:
    238            raise ValueError("No phase images have matching masks and dimensions.")
    239        return matched
    240    def _normalize_channel_list(self, channel_list, argument_name="channel_list"):
    241        if channel_list is None:
    242            raise ValueError(
    243                f"No {argument_name} provided. It should be of format ['channelname1', 'channelname2']."
    244            )
    245        if isinstance(channel_list, str):
    246            raise TypeError(
    247                f"{argument_name} must be an iterable of channel names, not a single string."
    248            )
    249        normalized_channels = list(channel_list)
    250        if not normalized_channels:
    251            raise ValueError(f"{argument_name} must contain at least one channel name.")
    252        return normalized_channels
    253
    254    def _iter_processing_targets(self, object_list, phase_channel):
    255        if isinstance(object_list, Image):
    256            return [object_list]
    257        if object_list is None:
    258            self.create_image_objects(phase_channel=phase_channel)
    259            return self.image_objects
    260        if isinstance(object_list, (list, tuple)):
    261            return list(object_list)
    262        raise TypeError(
    263            "object_list must be None, a single Image instance, or a list/tuple of Image instances."
    264        )
    265
    266    def _get_mesh_dataframe_lookup(self):
    267        """Build an image-name lookup from the current mesh dataframe.
    268
    269        Do not retain this lookup across calls: callers can replace or mutate
    270        ``mesh_df_collection``, and a cached lookup could then return stale
    271        meshes for newly loaded data.
    272        """
    273        if not isinstance(self.mesh_df_collection, pd.DataFrame) or self.mesh_df_collection.empty:
    274            return None
    275        return {
    276            image_name: group.reset_index(drop=True)
    277            for image_name, group in self.mesh_df_collection.groupby(
    278                "image_name", sort=False
    279            )
    280        }
    281
    282
    283    def load_masks(self):
    284        """
    285        Read masks from the ``/masks`` subfolder.
    286
    287        Mutates ``self.masks`` and ``self.mask_filenames``.
    288
    289        """
    290        self._require_image_folder_path()
    291        self.masks, self.mask_filenames = u.read_tiff_folder(
    292            self.image_folder_path + "/masks"
    293        )
    294        self._record_stage_success("load_masks")
    295
    296    def load_phase_images(self, phase_channel=""):
    297        """
    298        This method loads phase contrast images from the specified image folder path, based on the characters before the file-extension,
    299        and stores them in the 'self.images' array.
    300
    301        Parameters
    302        ----------
    303        phase_channel : str, optional
    304            Characters before the file-extension (e.g., "c1", "phase") to look for when using this function. (default is an empty string)
    305
    306        Mutates ``self.images``, ``self.image_filenames`` and ``self.paths``.
    307
    308        """
    309        self._require_image_folder_path()
    310        self.phase_channel = phase_channel or ""
    311        try:
    312            self.images, self.image_filenames, self.paths = u.read_tiff_folder(
    313                self.image_folder_path, phase_channel, include_paths=True
    314            )
    315        except Exception as e:
    316            raise type(e)(
    317                f"Failed to load phase contrast images from '{self.image_folder_path}' "
    318                f"with phase_channel='{phase_channel}': {e}"
    319            ) from e
    320        self._record_stage_success("load_phase_images")
    321
    322    def load_channel_images(self, channel_list):
    323        """
    324        This method loads channel images from the image folder path based on the specified characters before the file-extension and stores them in
    325        the 'self.channel_images' dictionary, where the channel name is used as the key.
    326
    327        Parameters
    328        ----------
    329        channel_list : List of str
    330            Characters before the extension (e.g., ["c2", "c3"]) to look for when using this function.
    331
    332        Mutates ``self.channel_images``.
    333
    334        """
    335        self._require_image_folder_path()
    336        self.channel_list = self._normalize_channel_list(channel_list)
    337        for channel in self.channel_list:
    338            # A failed read must not leave arrays from an earlier folder/load in place.
    339            self.channel_images.pop(channel, None)
    340            self.channel_image_filenames.pop(channel, None)
    341            self.channel_images_by_field.pop(channel, None)
    342            try:
    343                images, filenames = u.read_tiff_folder(
    344                    self.image_folder_path, suffix=channel
    345                )
    346            except Exception as e:
    347                self._record_processing_error(
    348                    "load_channel_images", e, channel=channel
    349                )
    350                continue
    351
    352            self.channel_images[channel] = list(images)
    353            self.channel_image_filenames[channel] = list(filenames)
    354            keyed_images = {}
    355            for filename, image in zip(filenames, images):
    356                key = u.image_field_key(filename, suffix=channel)
    357                if key in keyed_images:
    358                    self._record_processing_error(
    359                        "load_channel_images",
    360                        ValueError(f"Duplicate channel field identity '{key}'."),
    361                        image_name=filename,
    362                        channel=channel,
    363                    )
    364                    continue
    365                keyed_images[key] = image
    366            self.channel_images_by_field[channel] = keyed_images
    367        self._record_stage_success("load_channel_images")
    368
    369    def create_image_objects(self, phase_channel=""):
    370        """
    371        Create ``Image`` objects from the loaded masks and phase images.
    372
    373        Invalid collection-level setup raises immediately. Failures while creating
    374        a single image object are logged and skipped so the rest of the batch can continue.
    375        """
    376        if self.segmentation_failed:
    377            raise RuntimeError(
    378                "Segmentation failed in this collection; refusing to load masks "
    379                "from disk because they may belong to an earlier run."
    380            )
    381        if self.masks is None:
    382            self.load_masks()
    383        if self.images is None or self.image_filenames is None:
    384            self.load_phase_images(phase_channel)
    385
    386        matched_images = self._validate_phase_image_state()
    387
    388        # Feature tables and merged output belong to the previous image/cell set.
    389        self.feature_dataframes = {}
    390        self.merged_features = None
    391        self.image_objects = []
    392        mesh_lookup = self._get_mesh_dataframe_lookup()
    393
    394        for i, image_name, image, mask in matched_images:
    395            try:
    396                mesh_df = mesh_lookup.get(image_name) if mesh_lookup is not None else None
    397
    398                img_obj = self.create_image_object(
    399                    image, image_name, i, mask, mesh_df, self.px
    400                )
    401                self.image_objects.append(img_obj)
    402            except Exception as e:
    403                bactoscoop_logger.debug(traceback.format_exc())
    404                self._record_processing_error(
    405                    "create_image_objects", e, image_name=image_name
    406                )
    407        self._record_stage_success("create_image_objects")
    408
    409    def add_channels(self, img_objects, channel_list, load_data=True):
    410        """
    411        This method loads channel images based on the provided channel list and adds them to the individual image objects.
    412
    413        Parameters
    414        ----------
    415        img_objects : List
    416            List of image objects.
    417        channel_list : List
    418            List of channel names.
    419
    420        """
    421        channel_list = self._normalize_channel_list(channel_list)
    422        if img_objects is None or len(img_objects) == 0:
    423            raise ValueError("img_objects must contain at least one image object.")
    424
    425        # Load requested channels while keeping field identity beside each array.
    426        if load_data:
    427            self.load_channel_images(channel_list)
    428
    429        expected_fields = {
    430            u.image_field_key(obj.image_name, suffix=self.phase_channel)
    431            for obj in img_objects
    432        }
    433        for channel in channel_list:
    434            for extra_field in self.channel_images_by_field.get(channel, {}):
    435                if extra_field not in expected_fields:
    436                    self._record_processing_error(
    437                        "match_channel_image",
    438                        ValueError(
    439                            f"No phase image object matches {channel} field '{extra_field}'."
    440                        ),
    441                        image_name=extra_field,
    442                        channel=channel,
    443                    )
    444
    445        for img_obj in img_objects:
    446            channel_single_image_dict = dict(getattr(img_obj, "channels", None) or {})
    447            field_key = u.image_field_key(
    448                img_obj.image_name, suffix=self.phase_channel
    449            )
    450            for channel in channel_list:
    451                # Never retain an earlier image or derived interpolation for a
    452                # requested channel if its current file is missing or invalid.
    453                channel_single_image_dict.pop(channel, None)
    454                for cache_name in ("bg_channels", "chann_interp2d"):
    455                    cache = getattr(img_obj, cache_name, None)
    456                    if isinstance(cache, dict):
    457                        cache.pop(channel, None)
    458                channel_fields = self.channel_images_by_field.get(channel, {})
    459                channel_image = channel_fields.get(field_key)
    460                if channel_image is None:
    461                    self._record_processing_error(
    462                        "match_channel_image",
    463                        ValueError(
    464                            f"No {channel} image matches field '{img_obj.image_name}'."
    465                        ),
    466                        image_name=img_obj.image_name,
    467                        channel=channel,
    468                    )
    469                    continue
    470                if channel_image.ndim != 2 or img_obj.image.ndim != 2:
    471                    self._record_processing_error(
    472                        "match_channel_image",
    473                        ValueError(
    474                            f"{channel} and phase images must be 2D; got channel "
    475                            f"shape {channel_image.shape} and phase shape {img_obj.image.shape}."
    476                        ),
    477                        image_name=img_obj.image_name,
    478                        channel=channel,
    479                    )
    480                    continue
    481                if channel_image.shape != img_obj.image.shape:
    482                    self._record_processing_error(
    483                        "match_channel_image",
    484                        ValueError(
    485                            f"{channel} image shape {channel_image.shape} does not "
    486                            f"match phase shape {img_obj.image.shape}."
    487                        ),
    488                        image_name=img_obj.image_name,
    489                        channel=channel,
    490                    )
    491                    continue
    492                channel_single_image_dict[channel] = channel_image
    493            img_obj.channels = channel_single_image_dict or None
    494        self._record_stage_success("add_channels")
    495
    496    def get_mesh_dataframe(self, image_name):
    497        """
    498        This method returns the mesh dataframe associated with the specified image name.
    499
    500        Parameters
    501        ----------
    502        image_name : str
    503            The name or identifier of the image.
    504
    505        """
    506
    507        mesh_lookup = self._get_mesh_dataframe_lookup()
    508        if mesh_lookup is None:
    509            return None
    510        return mesh_lookup.get(image_name)
    511
    512    def create_image_object(self, image, image_name, index, mask, mesh_df, px):
    513        """
    514        This method creates a single Image object using the provided image, image name, index, mask, mesh dataframe, and pixel size (px).
    515        If mesh data is present, it also creates associated Cell objects.
    516
    517        Parameters
    518        ----------
    519        image : phase contrast image
    520            The phase contrast image for the Image object.
    521        image_name : str
    522            The name or identifier of the image.
    523        index : int
    524            The index or position of the image.
    525        mask : mask or None
    526            The mask or region of interest associated with the image.
    527        mesh_df : pd.DataFrame or None
    528            The mesh dataframe associated with the image, if available, or None.
    529        px : float
    530        The pixel size in micrometers.
    531        """
    532
    533        img_obj = Image(image, image_name, index, mask, mesh_df, px=px)
    534
    535        if mesh_df is not None:
    536            img_obj.create_cell_object(verbose=False)
    537        return img_obj
    538
    539    # Methods for batch processing of images ----------------------------------------------
    540    def segment_images(self, mask_thresh, minsize, n, model_name="bact_phase_omni"):
    541        self.segmentation_failed = False
    542        self.masks = None
    543        self.mask_filenames = None
    544        try:
    545            from .omni import Omnipose
    546            import torch
    547
    548            omni = Omnipose(self.images, self.paths)
    549            omni.load_models(model_name)
    550
    551            omni.compiled_process(n, mask_thresh, minsize)
    552            torch.cuda.empty_cache()
    553
    554            del omni
    555            gc.collect()
    556            self._record_stage_success("segment_images")
    557            return True
    558        except Exception as e:
    559            self.segmentation_failed = True
    560            bactoscoop_logger.debug(traceback.format_exc())
    561            self._record_processing_error("segment_images", e)
    562            return False
    563
    564    def batch_detect_objects(
    565        self,
    566        channels=None,
    567        reset_channels=True,
    568        align=False,
    569        smoothing=0.1,
    570        log_sigma=3,
    571        kernel_width=4,
    572        min_overlap_ratio=0.01,
    573        max_external_ratio=0.1,
    574    ):
    575        """
    576        Detect objects within the contour of the cell.
    577
    578        Parameters
    579        ----------
    580        channels : List of strings, optional
    581            The channel names of the channels to perform object detection on.
    582
    583        log_sigma : int, optional
    584            The sigma parameter of the Laplacian of Gaussian filter used in object detection. (default is 3)
    585
    586        kernel_width : int, optional
    587            The kernel width of the kernel used for dilating the object masks.
    588            Decreasing results in smaller masks, while increasing results in larger masks. (default is 4)
    589
    590        min_overlap_ratio : float, optional
    591            Minimum overlap between the object and cell required for the object to be kept. (default is 0.01)
    592
    593        max_external_ratio : float, optional
    594            The maximum ratio an object is allowed to lie outside the cell contour. (default is 0.1)
    595
    596        Returns
    597        -------
    598        pd.DataFrame
    599            A DataFrame containing cell_id, object_contours, and frame information.
    600
    601        """
    602        self._require_loaded_images()
    603        channels = self._normalize_channel_list(channels, argument_name="channels")
    604
    605        if reset_channels:
    606            self.channel_images = {}
    607            self.channel_image_filenames = {}
    608            self.channel_images_by_field = {}
    609            for image in self.image_objects:
    610                image.channels = None
    611                for cache_name in ("bg_channels", "chann_interp2d"):
    612                    cache = getattr(image, cache_name, None)
    613                    if isinstance(cache, dict):
    614                        cache.clear()
    615
    616        channels_to_load = [
    617            channel for channel in channels
    618            if channel not in self.channel_images_by_field
    619        ]
    620        if channels_to_load:
    621            self.load_channel_images(channels_to_load)
    622        self.add_channels(self.image_objects, channels, load_data=False)
    623
    624        bactoscoop_logger.info("Detecting objects within cell ...")
    625
    626        dfs = []
    627        for image in tqdm_if_verbose(self.image_objects):
    628            for cell in getattr(image, "cells", []):
    629                for channel in channels:
    630                    cell.object_meshdata.pop(channel, None)
    631            if image.channels is None or any(
    632                channel not in image.channels for channel in channels
    633            ):
    634                # The missing field has already been recorded by add_channels.
    635                continue
    636            try:
    637                detection_df = image.object_detection(
    638                    channels,
    639                    smoothing,
    640                    align,
    641                    log_sigma,
    642                    kernel_width,
    643                    min_overlap_ratio,
    644                    max_external_ratio,
    645                )
    646            except Exception as e:
    647                bactoscoop_logger.debug(traceback.format_exc())
    648                self._record_processing_error(
    649                    "object_detection", e, image_name=image.image_name
    650                )
    651                continue
    652            dfs.append(detection_df)
    653            cell_errors = getattr(image, "object_detection_errors", [])
    654            for cell_error in cell_errors:
    655                self._record_cell_error(
    656                    "cell_object_detection",
    657                    RuntimeError(cell_error["error"]),
    658                    image_name=image.image_name,
    659                    channel=cell_error.get("channel"),
    660                    cell_id=cell_error.get("cell_id"),
    661                    error_type=cell_error.get("error_type"),
    662                )
    663            cell_count = len(getattr(image, "cells", []))
    664            failed_attempts = sum(
    665                not error.get("recovered", False) for error in cell_errors
    666            )
    667            if cell_count and failed_attempts >= cell_count * len(channels):
    668                self._record_processing_error(
    669                    "object_detection",
    670                    ValueError(
    671                        f"All cell object-detection attempts failed for '{image.image_name}'."
    672                    ),
    673                    image_name=image.image_name,
    674                )
    675
    676        self.object_detection_df = (
    677            pd.concat(dfs, axis=0, ignore_index=True) if dfs else pd.DataFrame()
    678        )
    679
    680        if not self.object_detection_df.empty and "channel" in self.object_detection_df:
    681            bactoscoop_logger.info(self.object_detection_df["channel"].value_counts())
    682        else:
    683            bactoscoop_logger.info("No objects were detected.")
    684
    685        self._record_stage_success("batch_detect_objects")
    686        return self.object_detection_df
    687
    688    def batch_process_mesh(
    689        self,
    690        pkl_name=None,
    691        object_list=None,
    692        save_data=True,
    693        phase_channel="",
    694        join_thresh=4,
    695        split_thresh=0.35,
    696        CD_width=False,
    697        smoothing=0.1,
    698        neighbor_filter_max_neighbors=None,
    699        neighbor_filter_connectivity=1,
    700        max_daughter_cell_mesh_rows=800,
    701    ):
    702        """
    703        Process cellular meshes from segmented masks and store the data in Cell objects.
    704
    705        After segmentation, this method can process the resulting masks and create cellular meshes from them.
    706        It first joins cells with a pole-pole distance (px) below the 'join_thresh'.
    707        Subsequently, meshes of all cells are created and cells can be split again if the constriction degree exceeds the 'split_thresh'.
    708        The data is then stored in Cell objects and can be exported in a pickle file.
    709
    710        Parameters
    711        ----------
    712        pkl_name : str, optional
    713            Name of the pickle file to save the mesh data. If None, the file will be saved as 'folder_name_meshdata.pkl'.
    714
    715        object_list : List, optional
    716            Specify a list of Image objects to be processed. If 'object_list' is None, all Image instances in 'image_objects' are processed.
    717
    718        phase_channel : str, optional
    719            Characters before the file-extension (.tif, .tiff) that the algorithm looks for when using this function (e.g., "c1", "phase").
    720
    721        join_thresh : int, optional
    722            The pole-pole distance threshold for joining cells. Cells with a distance below this threshold will have their masks joined. (default is 4)
    723
    724        split_thresh : float, optional
    725            The constriction degree threshold for splitting cells. Cells with a constriction degree exceeding this threshold will be split. (default is 0.35)
    726
    727        max_daughter_cell_mesh_rows : int, optional
    728            Maximum rebuilt daughter mesh rows (default 800, minimum 4).
    729            Oversized resampling is rejected before allocation; both final
    730            daughters must have 4..limit rows. This is separate from the feature
    731            extraction max_mesh_size parameter and does not limit unsplit parents.
    732
    733        """
    734        max_daughter_cell_mesh_rows = u._validate_max_daughter_cell_mesh_rows(
    735            max_daughter_cell_mesh_rows
    736        )
    737        self.mesh_df_collection = pd.DataFrame()
    738        self.mesh_prefilter_stats = pd.DataFrame()
    739        object_list = self._iter_processing_targets(object_list, phase_channel)
    740        mesh_frames = []
    741        prefilter_stats = []
    742
    743        for img_obj in object_list:
    744            bactoscoop_logger.info(f"PROCESSING IMAGE: {img_obj.image_name}")
    745            try:
    746                img_obj.join_split_pipeline(
    747                    join_thresh,
    748                    split_thresh,
    749                    CD_width,
    750                    smoothing,
    751                    neighbor_filter_max_neighbors=neighbor_filter_max_neighbors,
    752                    neighbor_filter_connectivity=neighbor_filter_connectivity,
    753                    max_daughter_cell_mesh_rows=max_daughter_cell_mesh_rows,
    754                )
    755                mesh_df = getattr(img_obj, "processed_mesh_dataframe", None)
    756                if mesh_df is None:
    757                    raise ValueError(
    758                        f"Image '{img_obj.image_name}' did not produce a processed mesh dataframe."
    759                    )
    760                mesh_frames.append(mesh_df)
    761                if getattr(img_obj, "prefilter_summary", None) is not None:
    762                    prefilter_stats.append(
    763                        {
    764                            "image_name": img_obj.image_name,
    765                            **img_obj.prefilter_summary,
    766                        }
    767                    )
    768                img_obj.create_cell_object(verbose=False)
    769            except Exception as e:
    770                bactoscoop_logger.debug(traceback.format_exc())
    771                self._record_processing_error(
    772                    "mesh_processing", e, image_name=img_obj.image_name
    773                )
    774
    775        self.mesh_df_collection = (
    776            pd.concat(mesh_frames, ignore_index=True) if mesh_frames else pd.DataFrame()
    777        )
    778        self.mesh_prefilter_stats = (
    779            pd.DataFrame(prefilter_stats) if prefilter_stats else pd.DataFrame()
    780        )
    781
    782        if save_data:
    783            if pkl_name is not None:
    784                self._to_pickle(pkl_name, self.mesh_df_collection)
    785
    786            self._to_pickle(f"{self.name}_meshdata.pkl", self.mesh_df_collection)
    787
    788        self._record_stage_success("batch_process_mesh")
    789
    790
    791    def batch_load_mesh(self, pkl_name, pkl_path=None, phase_channel=""):
    792        """
    793        Load mesh data from a pickle-file and create image objects from the data.
    794
    795        Parameters
    796        ----------
    797        pkl_name : str
    798            Name of the pickle file where the mesh data is stored (e.g., 'Ecoli_meshdata.pkl').
    799
    800        pkl_path : str, optional
    801            Path to the file where the mesh data is stored. If not provided, the method will look in the image folder path.
    802
    803        phase_channel : str, optional
    804            Characters before the file-extension (.tif, .tiff) that the algorithm looks for when using this function (e.g., "c1", "phase").
    805
    806        """
    807        self.mesh_df_collection = None
    808        if pkl_path is None:
    809            pkl_path = self.image_folder_path
    810
    811        full_path = os.path.join(pkl_path, pkl_name)
    812        if not os.path.isfile(full_path):
    813            raise FileNotFoundError(f"Pickle file not found: {full_path}")
    814
    815        if full_path.lower().endswith(".parquet"):
    816            mesh_df = pd.read_parquet(full_path, engine="pyarrow")
    817            for column in ("contour", "mesh", "midline"):
    818                if column not in mesh_df:
    819                    raise ValueError(f"Mesh Parquet file is missing '{column}'.")
    820                mesh_df[column] = mesh_df[column].map(
    821                    lambda value: np.asarray(value.tolist(), dtype=float)
    822                )
    823            self.mesh_df_collection = mesh_df
    824        else:
    825            with open(full_path, "rb") as f:
    826                self.mesh_df_collection = pickle.load(f)
    827
    828        bactoscoop_logger.info("Meshes loaded from file: %s", pkl_name)
    829        self.create_image_objects(phase_channel=phase_channel)
    830
    831    def batch_mask2mesh(
    832        self,
    833        pkl_name=None,
    834        object_list=None,
    835        save_data=True,
    836        phase_channel="",
    837        smoothing=0.1,
    838    ):
    839        """
    840        Create meshes from masks without the joining or splitting as in the batch_process_mesh method.
    841
    842        Parameters
    843        ----------
    844        pkl_name : str, optional
    845            Name of the pickle file to save the mesh data. If None, the file will be saved as 'folder_name_meshdata.pkl'.
    846        object_list : List, optional
    847            Specify a list of Image objects to be processed. If 'object_list' is None, all Image instances in 'image_objects' are processed.
    848        phase_channel : str, optional
    849            Characters before the file-extension (.tif, .tiff) that the algorithm looks for when using this function (e.g., "c1", "phase").
    850        """
    851        self.mesh_df_collection = pd.DataFrame()
    852        mesh_frames = []
    853
    854        if isinstance(object_list, Image):
    855            print("\n")
    856            bactoscoop_logger.info(f"PROCESSING IMAGE: {object_list.image_name}")
    857
    858            object_list.mask2mesh(smoothing)
    859            mesh_frames.append(getattr(object_list, "mesh_dataframe"))
    860            object_list.create_cell_object(verbose=False)
    861        else:
    862
    863            if object_list is None:
    864                self.create_image_objects(phase_channel=phase_channel)
    865                object_list = self.image_objects
    866
    867            for img_obj in object_list:
    868
    869                bactoscoop_logger.info(f"PROCESSING IMAGE: {img_obj.image_name}")
    870
    871                img_obj.mask2mesh()
    872                mesh_frames.append(getattr(img_obj, "mesh_dataframe"))
    873
    874                img_obj.create_cell_object(verbose=False)
    875
    876        self.mesh_df_collection = (
    877            pd.concat(mesh_frames, ignore_index=True) if mesh_frames else pd.DataFrame()
    878        )
    879
    880        if save_data:
    881            if pkl_name is not None:
    882                self._to_pickle(pkl_name, self.mesh_df_collection)
    883
    884            self._to_pickle(
    885                "{}_meshdata.pkl".format(self.name), self.mesh_df_collection
    886            )
    887
    888    def batch_shift_correction(
    889        self,
    890        shifted_channel,
    891        reset_channels=False,
    892        log_sigma=1.5,
    893        kernel_width=3,
    894        min_overlap_ratio=0.1,
    895        max_external_ratio=1,
    896        phase_log_sigma=0.5,
    897        phase_closing_level=2,
    898        signal_closing_level=12,
    899        max_shift_correction=15,
    900    ):
    901        self._require_loaded_images()
    902        shifted_channel = self._normalize_channel_list(
    903            shifted_channel, argument_name="shifted_channel"
    904        )
    905
    906        if reset_channels:
    907            self.channel_images = {}
    908            self.channel_image_filenames = {}
    909            self.channel_images_by_field = {}
    910            for image in self.image_objects:
    911                image.channels = None
    912                for cache_name in ("bg_channels", "chann_interp2d"):
    913                    cache = getattr(image, cache_name, None)
    914                    if isinstance(cache, dict):
    915                        cache.clear()
    916
    917        channels_to_load = [
    918            channel for channel in shifted_channel
    919            if channel not in self.channel_images_by_field
    920        ]
    921        if channels_to_load:
    922            self.load_channel_images(channels_to_load)
    923        self.add_channels(self.image_objects, shifted_channel, load_data=False)
    924
    925        success_percentage_list = []
    926        bactoscoop_logger.info("Aligning contour, mesh and midline to channel image ...")
    927        for image_obj in tqdm_if_verbose(self.image_objects):
    928            if image_obj.channels is None or any(
    929                channel not in image_obj.channels for channel in shifted_channel
    930            ):
    931                # Missing channels were recorded by add_channels.
    932                continue
    933            try:
    934                success_percentage = image_obj.shift_correction(
    935                    shifted_channel,
    936                    log_sigma,
    937                    kernel_width,
    938                    min_overlap_ratio,
    939                    max_external_ratio,
    940                    phase_log_sigma,
    941                    phase_closing_level,
    942                    signal_closing_level,
    943                    max_shift_correction,
    944                )
    945            except Exception as e:
    946                bactoscoop_logger.debug(traceback.format_exc())
    947                self._record_processing_error(
    948                    "shift_correction", e, image_name=image_obj.image_name
    949                )
    950                continue
    951            success_percentage_list.append(success_percentage)
    952            cell_errors = getattr(image_obj, "shift_errors", [])
    953            for cell_error in cell_errors:
    954                self._record_cell_error(
    955                    "cell_shift_correction",
    956                    RuntimeError(cell_error["error"]),
    957                    image_name=image_obj.image_name,
    958                    channel=cell_error.get("channel"),
    959                    cell_id=cell_error.get("cell_id"),
    960                    error_type=cell_error.get("error_type"),
    961                )
    962            cell_count = len(getattr(image_obj, "cells", []))
    963            if cell_count and len(cell_errors) >= cell_count * len(shifted_channel):
    964                self._record_processing_error(
    965                    "shift_correction",
    966                    ValueError(
    967                        f"All cell shift-correction attempts failed for '{image_obj.image_name}'."
    968                    ),
    969                    image_name=image_obj.image_name,
    970                )
    971        if not success_percentage_list:
    972            bactoscoop_logger.warning("No shift-correction results were produced.")
    973            return 0.0
    974
    975        average_success_percentage = sum(success_percentage_list) / len(
    976            success_percentage_list
    977        )
    978
    979        bactoscoop_logger.info(f"Shift succes rate: {average_success_percentage}")
    980        return average_success_percentage
    981
    982    def batch_calculate_features(
    983        self,
    984        channel_method_pairs,
    985        all_data=True,
    986        reset=True,
    987        use_shifted_contours=False,
    988        shift_signal=False,
    989        max_mesh_size=600,
    990        retain_contour_on_object_mesh_failure_channels=None,
    991    ):
    992        """
    993        Calculate features of all image objects for specified channels and methods.
    994
    995        This method calculates features for all image objects based on the specified channel-method pairs. It takes a list of tuples, where each tuple contains channel names and the feature calculation method. The resulting feature dataframes are stored in a dictionary.
    996
    997        Parameters
    998        ----------
    999        channel_method_pairs : List[Tuple[List[str], str]]
   1000            A list of tuples where each tuple contains the channel names and feature calculation method.
   1001            Example of channel_method_pairs: [(['c1'], 'profiling'), (['c2', 'c3'], 'objects'), ([None], 'svm')].
   1002            With the example pairs, this method will analyze c1 using the profiling method,
   1003            then c2 and c3 using the objects method, and finally the phase contrast channel using the svm method.
   1004
   1005        add_profiling_data : bool, optional
   1006            Whether to include profiling data in the feature calculation. (default is True)
   1007
   1008        reset : bool, optional
   1009            If True, the dictionary storing the dataframes will be reset. If False, dataframes are added to the existing dictionary. (default is True)
   1010        retain_contour_on_object_mesh_failure_channels : list[str] or None, optional
   1011            Channels whose objects features retain contour-based measurements
   1012            when any detected object mesh is invalid. Other channels omit that
   1013            cell's objects feature row. None is equivalent to an empty list.
   1014            This option does not affect other feature methods.
   1015
   1016        Returns
   1017        -------
   1018        Dict
   1019            A dictionary containing dataframes with calculated features.
   1020
   1021        """
   1022        self._require_loaded_images()
   1023        if channel_method_pairs is None:
   1024            raise ValueError("channel_method_pairs must be provided.")
   1025        if retain_contour_on_object_mesh_failure_channels is None:
   1026            retained_channels = frozenset()
   1027        elif isinstance(retain_contour_on_object_mesh_failure_channels, str):
   1028            raise TypeError(
   1029                "retain_contour_on_object_mesh_failure_channels must be a list "
   1030                "of channel names, not a string."
   1031            )
   1032        else:
   1033            retained_channels = frozenset(
   1034                retain_contour_on_object_mesh_failure_channels
   1035            )
   1036        if not all(isinstance(channel, str) for channel in retained_channels):
   1037            raise TypeError(
   1038                "retain_contour_on_object_mesh_failure_channels must contain "
   1039                "only channel names."
   1040            )
   1041
   1042        self.merged_features = None
   1043        if reset:
   1044            self.feature_dataframes = {}
   1045
   1046        for channels, method in channel_method_pairs:
   1047            normalized_channels = (
   1048                [None]
   1049                if channels == [None]
   1050                else self._normalize_channel_list(channels, argument_name="channels")
   1051            )
   1052            for channel in normalized_channels:
   1053                key = f"{method}_{channel}_features"
   1054                self.feature_dataframes.pop(key, None)
   1055                bactoscoop_logger.info(
   1056                    f"PROCESSING CHANNEL: {channel}, method: {method}"
   1057                )
   1058                feature_dfs = []
   1059
   1060                for img_obj in tqdm_if_verbose(self.image_objects):
   1061                    if channel is not None and (
   1062                        img_obj.channels is None or channel not in img_obj.channels
   1063                    ):
   1064                        self._record_processing_error(
   1065                            "image_feature_calculation",
   1066                            ValueError(
   1067                                f"Channel {channel} is not loaded for image '{img_obj.image_name}'."
   1068                            ),
   1069                            image_name=img_obj.image_name,
   1070                            channel=channel,
   1071                        )
   1072                        continue
   1073                    try:
   1074                        result_df = img_obj.calculate_features(
   1075                            method,
   1076                            channel,
   1077                            all_data,
   1078                            use_shifted_contours,
   1079                            shift_signal,
   1080                            max_mesh_size,
   1081                            retained_channels,
   1082                        )
   1083                    except Exception as e:
   1084                        bactoscoop_logger.debug(traceback.format_exc())
   1085                        self._record_processing_error(
   1086                            "image_feature_calculation",
   1087                            e,
   1088                            image_name=img_obj.image_name,
   1089                            channel=channel,
   1090                        )
   1091                        continue
   1092                    if result_df is not None:
   1093                        feature_dfs.append(result_df)
   1094                    for feature_error in getattr(img_obj, "feature_errors", []):
   1095                        self._record_cell_error(
   1096                            "cell_feature_calculation",
   1097                            RuntimeError(feature_error["error"]),
   1098                            image_name=img_obj.image_name,
   1099                            channel=channel,
   1100                            cell_id=feature_error.get("cell_id"),
   1101                            error_type=feature_error.get("error_type"),
   1102                        )
   1103
   1104                if feature_dfs:
   1105                    concatenated_df = pd.concat(feature_dfs, axis=0)
   1106                    concatenated_df.reset_index(drop=True, inplace=True)
   1107                    self.feature_dataframes[key] = concatenated_df
   1108                    has_rows = not concatenated_df.empty
   1109                else:
   1110                    has_rows = False
   1111                if not has_rows:
   1112                    self._record_processing_error(
   1113                        "batch_calculate_features",
   1114                        ValueError(
   1115                            f"No feature rows were produced for channel {channel}, method {method}."
   1116                        ),
   1117                        channel=channel,
   1118                    )
   1119
   1120        self._record_stage_success("batch_calculate_features")
   1121        return self.feature_dataframes
   1122
   1123    def batch_calculate_signal_correlation_features(
   1124        self, df, channels, feature_method_tuples
   1125    ):
   1126        from .signalcorrelation import SignalCorrelation
   1127
   1128        # Generate all unique pairs of channels
   1129        channel_pairs = combinations(channels, 2)
   1130
   1131        for channel1, channel2 in channel_pairs:
   1132            for features, method_names in feature_method_tuples:
   1133                bactoscoop_logger.info(
   1134                    f"Processing channel {channel1} vs {channel2} and features {features} from {self.image_folder_path}:"
   1135                )
   1136                for feature in tqdm_if_verbose(features):
   1137                    prepared_feature = SignalCorrelation.prepare_feature_pair(
   1138                        df, channel1, channel2, feature
   1139                    )
   1140
   1141                    for method_name in method_names:
   1142                        sc = SignalCorrelation(
   1143                            df,
   1144                            channel1,
   1145                            channel2,
   1146                            feature,
   1147                            method_name,
   1148                            prepared_feature=prepared_feature,
   1149                        )
   1150                        sc.calculate()
   1151                        del sc
   1152        return df
   1153
   1154    def _apply_curation_labels(self):
   1155        """Apply curation labels by stable image identity, with frame fallback."""
   1156        image_objects_by_name = {}
   1157        image_objects_by_frame = {}
   1158        for image in self.image_objects:
   1159            image_objects_by_name.setdefault(image.image_name, []).append(image)
   1160            image_objects_by_frame.setdefault(image.frame, []).append(image)
   1161
   1162        for _, row in self.curated_df.iterrows():
   1163            image_name = row.get("image_name")
   1164            cell_id = row["cell_id"]
   1165            label = row["label"]
   1166            image = None
   1167
   1168            if pd.notna(image_name):
   1169                matches = image_objects_by_name.get(image_name, [])
   1170                if len(matches) == 1:
   1171                    image = matches[0]
   1172                else:
   1173                    reason = (
   1174                        f"Image name '{image_name}' is not unique."
   1175                        if len(matches) > 1
   1176                        else f"No image object matches image name '{image_name}'."
   1177                    )
   1178                    self._record_processing_error(
   1179                        "curation_mapping",
   1180                        ValueError(reason),
   1181                        image_name=image_name,
   1182                        cell_id=cell_id,
   1183                    )
   1184                    continue
   1185
   1186            if image is None:
   1187                frame = row.get("frame")
   1188                matches = image_objects_by_frame.get(frame, [])
   1189                if len(matches) == 1:
   1190                    image = matches[0]
   1191                else:
   1192                    self._record_processing_error(
   1193                        "curation_mapping",
   1194                        ValueError(
   1195                            f"Could not uniquely map curated cell {cell_id} to image "
   1196                            f"(image_name={image_name!r}, frame={frame!r})."
   1197                        ),
   1198                        image_name=image_name if pd.notna(image_name) else None,
   1199                        cell_id=cell_id,
   1200                    )
   1201                    continue
   1202
   1203            frame = row.get("frame")
   1204            if pd.notna(frame) and image.frame != frame:
   1205                self._record_processing_error(
   1206                    "curation_mapping",
   1207                    ValueError(
   1208                        f"Curated cell {cell_id} names image '{image.image_name}' "
   1209                        f"but has frame {frame}; expected {image.frame}."
   1210                    ),
   1211                    image_name=image.image_name,
   1212                    cell_id=cell_id,
   1213                )
   1214                continue
   1215
   1216            cells = [cell for cell in image.cells if cell.cell_id == cell_id]
   1217            if len(cells) != 1:
   1218                reason = (
   1219                    f"Cell ID {cell_id} was not found in image '{image.image_name}'."
   1220                    if not cells
   1221                    else f"Cell ID {cell_id} is not unique in image '{image.image_name}'."
   1222                )
   1223                self._record_processing_error(
   1224                    "curation_mapping",
   1225                    ValueError(reason),
   1226                    image_name=image.image_name,
   1227                    cell_id=cell_id,
   1228                )
   1229                continue
   1230            if label == 0:
   1231                image.cells.remove(cells[0])
   1232
   1233        self._record_stage_success("curate_dataset")
   1234
   1235    def curate_dataset(
   1236        self, path_to_model, cols=4, control=False, save_curated_data=True
   1237    ):
   1238        """
   1239        Curate the dataset based on a trained support vector machine model.
   1240
   1241        This method calculates SVM features and uses a trained SVM model to classify cells based on these features.
   1242        It curates the dataset by discarding cells that are classified as label 0 (non-interesting).
   1243        Optionally, it can include positive and negative control cells.
   1244
   1245        Parameters:
   1246            path_to_model (str): Path to the trained SVM model.
   1247
   1248            cols (int): Retained for backward compatibility; currently unused.
   1249
   1250            control (bool): Whether to include positive and negative control cells.
   1251
   1252            save_curated_data (bool): Whether to write the curated mesh pickle.
   1253                The in-memory mesh dataframe is updated either way.
   1254
   1255        Returns:
   1256            None
   1257        """
   1258        self.batch_calculate_features(
   1259            channel_method_pairs=[([None], "svm")], all_data=True, reset=True, max_mesh_size=800
   1260        )
   1261
   1262        from .curation import Curation
   1263
   1264        cur = Curation(self.feature_dataframes["svm_None_features"])
   1265
   1266        self.curated_df = cur.compiled_curation(path_to_model, cols)
   1267
   1268        p1, p0 = cur.get_label_proportions()
   1269        bactoscoop_logger.info(
   1270            f"\nProportion of label 1: {p1}\nProportion of label 0: {p0}"
   1271        )
   1272
   1273        if control:
   1274            import bactoscoop.plot as plotting
   1275
   1276            self.pos, self.neg = cur.get_control(5, 5)
   1277            if self.pos and self.neg is not None:
   1278                bactoscoop_logger.info(
   1279                    f"\nCell IDs and Frames for positive control are {self.pos}\nFrame and Cell IDs for negative control are {self.neg}"
   1280                )
   1281                plotting.plot_svm_controls(
   1282                    self.pos, self.image_objects, message="Positive"
   1283                )
   1284                plotting.plot_svm_controls(
   1285                    self.neg, self.image_objects, message="Negative"
   1286                )
   1287
   1288        self._apply_curation_labels()
   1289
   1290
   1291        # The in-memory mesh population follows the curated cells even when
   1292        # the caller is only testing and does not want a file overwritten.
   1293        self.add_meshdata_to_dataframe()
   1294        if save_curated_data:
   1295            self._to_pickle(
   1296                "{}_curated_meshdata.pkl".format(self.name), self.mesh_df_collection
   1297            )
   1298
   1299        del cur
   1300
   1301    def add_meshdata_to_dataframe(self):
   1302        data = []
   1303
   1304        # Iterate over each image object
   1305        for image in self.image_objects:
   1306            # Iterate over each cell in the current image
   1307            for cell in image.cells:
   1308                # Extract the required attributes from the cell and image
   1309                cell_id = cell.cell_id
   1310                contour = cell.contour
   1311                midline = cell.midline
   1312                mesh = cell.mesh
   1313                frame = image.frame
   1314                image_name = image.image_name
   1315
   1316                # Append the data as a dictionary to the list
   1317                data.append(
   1318                    {
   1319                        "image_name": image_name,
   1320                        "frame": frame,
   1321                        "cell_id": cell_id,
   1322                        "contour": contour,
   1323                        "midline": midline,
   1324                        "mesh": mesh,
   1325                    }
   1326                )
   1327
   1328        # Create DataFrame from the collected data
   1329        self.mesh_df_collection = pd.DataFrame(data)
   1330
   1331    def export_curated_masks(self, output_subfolder="curated_masks", overwrite=True):
   1332        """Write one label mask per image from the SVM-curated cell contours.
   1333
   1334        This method is intended to be run after :meth:`curate_dataset`. It never
   1335        changes the original Omnipose masks in ``masks``. Instead, it writes new
   1336        masks to ``output_subfolder`` and records how every curated label overlaps
   1337        the original labels, so the output can be used directly by BactoFeatures.
   1338
   1339        Parameters
   1340        ----------
   1341        output_subfolder : str, optional
   1342            Folder below ``image_folder_path`` for the exported label TIFFs.
   1343        overwrite : bool, optional
   1344            If ``False``, fail before replacing an existing curated-mask TIFF.
   1345
   1346        Returns
   1347        -------
   1348        tuple[list[str], pandas.DataFrame]
   1349            Paths to the exported masks and a provenance table with one row per
   1350            curated cell.
   1351        """
   1352        self._require_image_folder_path()
   1353        if not self.image_objects:
   1354            raise ValueError(
   1355                "No image objects are available. Load curated mesh data or run "
   1356                "curate_dataset() before exporting curated masks."
   1357            )
   1358
   1359        output_dir = os.path.join(self.image_folder_path, output_subfolder)
   1360        os.makedirs(output_dir, exist_ok=True)
   1361        mask_paths = []
   1362        provenance_rows = []
   1363
   1364        for image in self.image_objects:
   1365            if image.mask is None:
   1366                raise ValueError(f"Original mask is not loaded for {image.image_name}.")
   1367
   1368            max_label = len(image.cells)
   1369            output_dtype = np.uint16 if max_label <= np.iinfo(np.uint16).max else np.uint32
   1370            curated_mask = np.zeros(image.mask.shape, dtype=output_dtype)
   1371            stem = os.path.splitext(image.image_name)[0]
   1372            output_path = os.path.join(output_dir, f"{stem}_curated_masks.tif")
   1373            if os.path.exists(output_path) and not overwrite:
   1374                raise FileExistsError(
   1375                    f"Curated mask already exists: {output_path}. Set overwrite=True to replace it."
   1376                )
   1377
   1378            for curated_label, cell in enumerate(image.cells, start=1):
   1379                contour = np.asarray(cell.contour, dtype=float)
   1380                if contour.ndim != 2 or contour.shape[1] != 2 or len(contour) < 3:
   1381                    bactoscoop_logger.warning(
   1382                        "Skipping curated cell %s in %s: invalid contour.",
   1383                        cell.cell_id,
   1384                        image.image_name,
   1385                    )
   1386                    continue
   1387
   1388                cell_mask = np.zeros(image.mask.shape, dtype=np.uint8)
   1389                # Cell contours are (row, column); OpenCV expects (x, y).
   1390                polygon = np.rint(contour[:, ::-1]).astype(np.int32).reshape(-1, 1, 2)
   1391                cv.fillPoly(cell_mask, [polygon], color=1)
   1392                cell_pixels = cell_mask.astype(bool)
   1393                if not cell_pixels.any():
   1394                    bactoscoop_logger.warning(
   1395                        "Skipping curated cell %s in %s: contour has no in-image pixels.",
   1396                        cell.cell_id,
   1397                        image.image_name,
   1398                    )
   1399                    continue
   1400
   1401                curated_mask[cell_pixels] = curated_label
   1402                source_labels = np.unique(image.mask[cell_pixels])
   1403                source_labels = source_labels[source_labels != 0].astype(int).tolist()
   1404                rows, columns = np.nonzero(cell_pixels)
   1405                provenance_rows.append(
   1406                    {
   1407                        "image_name": image.image_name,
   1408                        "frame": image.frame,
   1409                        "curated_label": curated_label,
   1410                        "bactoscoop_cell_id": cell.cell_id,
   1411                        "centroid_x_px": float(columns.mean()),
   1412                        "centroid_y_px": float(rows.mean()),
   1413                        "area_px": int(cell_pixels.sum()),
   1414                        "original_mask_path": os.path.join(
   1415                            self.image_folder_path, "masks", f"{stem}_cp_masks.tif"
   1416                        ),
   1417                        "original_mask_labels": ";".join(map(str, source_labels)),
   1418                        "curated_mask_path": output_path,
   1419                    }
   1420                )
   1421
   1422            tifffile.imwrite(output_path, curated_mask)
   1423            mask_paths.append(output_path)
   1424
   1425        provenance = pd.DataFrame(provenance_rows)
   1426        provenance_path = os.path.join(output_dir, "curated_mask_provenance.csv")
   1427        provenance.to_csv(provenance_path, index=False)
   1428        bactoscoop_logger.info(
   1429            "Exported %s curated masks and %s curated cells to %s",
   1430            len(mask_paths),
   1431            len(provenance),
   1432            output_dir,
   1433        )
   1434        return mask_paths, provenance
   1435
   1436    def merge_dataframes(self, include_metadata_tag=False, discard_morphological_nan=False):
   1437        """
   1438        Merge feature dataframes into a single dataframe.
   1439    
   1440        Merges multiple feature dataframes into a single dataframe, ensuring that columns are properly renamed
   1441        to identify the channel and removes duplicates based on specific columns.
   1442    
   1443        Args:
   1444            include_metadata_tag (bool): Whether to rename 'image_name', 'cell_id', and 'frame' to 'Metadata_*'.
   1445            discard_nan (bool): Whether to discard rows with NaN in 'cell_area' and 'cell_length'.
   1446    
   1447        Returns:
   1448            pd.DataFrame: The merged feature dataframe.
   1449        """
   1450        if not self.feature_dataframes:
   1451            raise ValueError("No feature dataframes are available to merge.")
   1452
   1453        merged_dataframes = []
   1454
   1455        for key, df in self.feature_dataframes.items():
   1456            # Keys are "method_channel_features"; channel names may contain underscores.
   1457            if not key.endswith("_features"):
   1458                raise ValueError(f"Invalid feature dataframe key: {key!r}")
   1459            method_and_channel = key[: -len("_features")]
   1460            method, separator, channel = method_and_channel.partition("_")
   1461            if not separator or not method or not channel:
   1462                raise ValueError(f"Invalid feature dataframe key: {key!r}")
   1463
   1464            # Set a prefix to identify the channel in column names if the channel is not None
   1465            if channel != "None":
   1466                df = df.rename(
   1467                    columns={
   1468                        col: f"{channel}_{col}"
   1469                        for col in df.columns
   1470                        if col not in ["image_name", "cell_id", "frame"]
   1471                    }
   1472                )
   1473
   1474            # Append the modified dataframe to the list
   1475            merged_dataframes.append(df)
   1476
   1477        # Concatenate the dataframes vertically
   1478        merged_data = pd.concat(merged_dataframes)
   1479
   1480        # Merge the dataframes based on 'image_name', 'cell_id', and 'frame'
   1481        result = merged_data.pivot_table(
   1482            index=["image_name", "cell_id", "frame"], aggfunc="first"
   1483        ).reset_index()
   1484
   1485        # Discard rows with NaN in 'cell_area' and 'cell_length' if discard_nan is True
   1486        if discard_morphological_nan:
   1487            result = result.dropna(subset=["cell_area", "cell_length"])
   1488
   1489        if include_metadata_tag:
   1490            # Rename 'image_name', 'cell_id', and 'frame' to 'Metadata_*'
   1491            result = result.rename(
   1492                columns={
   1493                    "image_name": "Metadata_image_name",
   1494                    "cell_id": "Metadata_cell_id",
   1495                    "frame": "Metadata_frame"
   1496                }
   1497            )
   1498
   1499        # Reset the index to make it look like the final result
   1500        self.merged_features = result.reset_index(drop=True)
   1501
   1502        return self.merged_features
   1503
   1504    def dataframe_to_pkl(self, pkl_name=None):
   1505        """Save merged features in the image folder and return the pickle path.
   1506
   1507        A suffixless ``pkl_name`` is a legacy tag appended to the dataset name;
   1508        a name ending in ``.pkl`` is used as the exact filename.
   1509        """
   1510        if self.merged_features is None:
   1511            raise ValueError("Merge feature dataframes before saving them.")
   1512        if pkl_name is None:
   1513            filename = f"{self.name}_features.pkl"
   1514        elif str(pkl_name).endswith(".pkl"):
   1515            filename = str(pkl_name)
   1516        else:
   1517            filename = f"{self.name}_{pkl_name}.pkl"
   1518        self._to_pickle(filename, self.merged_features)
   1519        return os.path.join(self.image_folder_path, filename)
   1520
   1521    def meshdata_to_parquet(self, parquet_name=None):
   1522        """Save the current mesh dataframe as reloadable Parquet in the image folder.
   1523
   1524        Geometry arrays become nested numeric lists in Parquet. Reload with
   1525        ``batch_load_mesh("name.parquet", phase_channel=...)``.
   1526        Requires the optional ``pyarrow`` package.
   1527        """
   1528        if self.mesh_df_collection is None or self.mesh_df_collection.empty:
   1529            raise ValueError("No mesh data is available to save.")
   1530        name = parquet_name or f"{self.name}_meshdata.parquet"
   1531        self._to_parquet(name, self.mesh_df_collection, mesh_data=True)
   1532
   1533    def dataframe_to_parquet(self, parquet_name=None):
   1534        """Save merged features as Parquet in the image folder (requires pyarrow)."""
   1535        if self.merged_features is None:
   1536            raise ValueError("Merge feature dataframes before saving them.")
   1537        name = parquet_name or f"{self.name}_features.parquet"
   1538        self._to_parquet(name, self.merged_features)
   1539
   1540    def _atomic_write(self, filename, writer):
   1541        self._require_image_folder_path()
   1542        if os.path.basename(filename) != filename or filename in {"", ".", ".."}:
   1543            raise ValueError("Output filename must be a name within the image folder.")
   1544        file_path = os.path.join(self.image_folder_path, filename)
   1545        descriptor, temporary_path = tempfile.mkstemp(
   1546            prefix=f".{filename}.", suffix=".tmp", dir=self.image_folder_path
   1547        )
   1548        os.close(descriptor)
   1549        try:
   1550            writer(temporary_path)
   1551            with open(temporary_path, "rb+") as temporary_file:
   1552                os.fsync(temporary_file.fileno())
   1553            os.replace(temporary_path, file_path)
   1554        finally:
   1555            if os.path.exists(temporary_path):
   1556                os.unlink(temporary_path)
   1557
   1558    def _to_parquet(self, filename, data, mesh_data=False):
   1559        if not filename.lower().endswith(".parquet"):
   1560            raise ValueError("Parquet filename must end in .parquet")
   1561        output = data.copy()
   1562        if mesh_data:
   1563            for column in ("contour", "mesh", "midline"):
   1564                if column not in output:
   1565                    raise ValueError(f"Mesh dataframe is missing '{column}'.")
   1566                output[column] = output[column].map(
   1567                    lambda value: np.asarray(value).tolist()
   1568                )
   1569        self._atomic_write(
   1570            filename,
   1571            lambda path: output.to_parquet(
   1572                path, engine="pyarrow", compression="zstd", index=False
   1573            ),
   1574        )
   1575
   1576    def _to_pickle(self, pkl_name, data):
   1577        """
   1578        Save data to a pkl file.
   1579
   1580        """
   1581        self._atomic_write(
   1582            pkl_name,
   1583            lambda path: self._write_pickle_file(path, data),
   1584        )
   1585
   1586    @staticmethod
   1587    def _write_pickle_file(path, data):
   1588        with open(path, "wb") as file:
   1589            pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)