1# -*- coding: utf-8 -*-
2"""
3Created on Wed Apr 12 16:47:19 2023
4
5@author: Bart Steemans. Govers Lab.
6"""
7
8import os
9import pickle
10import cv2 as cv
11import numpy as np
12import tifffile
13from . import utilities as u
14from .logging_utils import get_bactoscoop_logger, tqdm_if_verbose
15from .image import Image
16import pandas as pd
17from itertools import combinations
18
19import logging, sys, datetime, traceback
20import gc
21import tempfile
22
23
24bactoscoop_logger = get_bactoscoop_logger()
25# # Change following to DEBUG to see more information
26# bactoscoop_logger.setLevel(logging.INFO)
27
28
29class ImageCollection:
30 """ """
31
32 def __init__(
33 self, image_folder_path=None, px=0.065, error_policy="continue",
34 log_cell_errors=False,
35 ):
36 """Create a collection.
37
38 ``error_policy="continue"`` records item-level errors and proceeds with
39 independent fields. ``error_policy="raise"`` raises on the first one.
40 Required-stage failures still stop dependent work in either mode.
41 Cell feature failures are recorded separately and do not make a run
42 partial in continue mode. Set ``log_cell_errors=True`` to print each one.
43 """
44 if error_policy not in {"continue", "raise"}:
45 raise ValueError("error_policy must be either 'continue' or 'raise'.")
46 self.error_policy = error_policy
47 self.log_cell_errors = log_cell_errors
48 self.processing_errors = []
49 self.cell_errors = []
50 self.completed_stages = set()
51
52 self.px = px # microns per pixel
53 self.images = None
54 self.masks = None
55 self.channel_images = {}
56 self.channel_image_filenames = {}
57 self.channel_images_by_field = {}
58 self.inverted_image = None
59 self.channel_list = None
60 self.image_filenames = None
61 self.mask_filenames = None
62 self.phase_channel = ""
63 self.segmentation_failed = False
64
65 self.image_folder_path = image_folder_path # Path to the folder to be processed
66 self.name = (
67 os.path.basename(os.path.normpath(self.image_folder_path))
68 if self.image_folder_path is not None
69 else None
70 )
71
72 self.image_objects = [] # Array that will be populated with objects
73
74 self.mesh_df_collection = pd.DataFrame()
75
76 self.feature_dataframes = {}
77 self.merged_features = None
78 self.mesh_prefilter_stats = pd.DataFrame()
79
80 def _require_image_folder_path(self):
81 if self.image_folder_path is None:
82 raise ValueError("image_folder_path must be set before loading image data.")
83
84 def _require_loaded_images(self):
85 if not self.image_objects:
86 raise ValueError(
87 "No image objects are available. Run create_image_objects() or load mesh data first."
88 )
89
90 def _record_processing_error(
91 self, stage, error, image_name=None, channel=None, cell_id=None
92 ):
93 record = {
94 "stage": stage,
95 "image_name": image_name,
96 "channel": channel,
97 "cell_id": cell_id,
98 "error_type": type(error).__name__,
99 "error": str(error),
100 }
101 self.processing_errors.append(record)
102 bactoscoop_logger.error(
103 "Processing error at %s (image=%s, channel=%s, cell_id=%s): %s",
104 stage,
105 image_name,
106 channel,
107 cell_id,
108 error,
109 )
110 if self.error_policy == "raise":
111 raise error
112 return record
113
114 def _record_cell_error(
115 self, stage, error, image_name, channel, cell_id, error_type=None
116 ):
117 record = {
118 "stage": stage,
119 "image_name": image_name,
120 "channel": channel,
121 "cell_id": cell_id,
122 "error_type": error_type or type(error).__name__,
123 "error": str(error),
124 }
125 self.cell_errors.append(record)
126 if self.log_cell_errors:
127 bactoscoop_logger.warning(
128 "Cell processing error at %s (image=%s, channel=%s, cell_id=%s): %s",
129 stage, image_name, channel, cell_id, error,
130 )
131 if self.error_policy == "raise":
132 raise error
133 return record
134
135 def processing_summary(self):
136 """Return attempted-stage status and retained image/cell error records."""
137 if self.processing_errors:
138 status = "completed_with_errors"
139 elif self.completed_stages:
140 status = "completed"
141 else:
142 status = "not_started"
143 return {
144 "status": status,
145 "completed_stages": sorted(self.completed_stages),
146 "error_count": len(self.processing_errors),
147 "errors": list(self.processing_errors),
148 "cell_error_count": len(self.cell_errors),
149 "cell_errors": list(self.cell_errors),
150 }
151
152 def _record_stage_success(self, stage):
153 self.completed_stages.add(stage)
154
155 def _validate_phase_image_state(self):
156 if self.images is None or self.image_filenames is None:
157 raise ValueError(
158 "Phase images are not loaded. Run load_phase_images() before creating image objects."
159 )
160 if self.masks is None:
161 raise ValueError(
162 "Masks are not loaded. Run load_masks() before creating image objects."
163 )
164 if len(self.image_filenames) != len(self.images):
165 raise ValueError(
166 f"Loaded {len(self.image_filenames)} image names but {len(self.images)} phase images."
167 )
168 if self.mask_filenames is None or len(self.mask_filenames) != len(self.masks):
169 raise ValueError("Mask arrays and mask filenames are not aligned.")
170
171 phase_by_key = {}
172 for index, filename in enumerate(self.image_filenames):
173 key = u.image_field_key(filename, suffix=self.phase_channel)
174 if key in phase_by_key:
175 self._record_processing_error(
176 "phase_mask_matching",
177 ValueError(f"Duplicate phase field identity '{key}'."),
178 image_name=filename,
179 )
180 continue
181 phase_by_key[key] = (index, filename, self.images[index])
182
183 masks_by_key = {}
184 for index, filename in enumerate(self.mask_filenames):
185 key = u.image_field_key(
186 filename, suffix=self.phase_channel, is_mask=True
187 )
188 if key in masks_by_key:
189 self._record_processing_error(
190 "phase_mask_matching",
191 ValueError(f"Duplicate mask field identity '{key}'."),
192 image_name=filename,
193 )
194 continue
195 masks_by_key[key] = (filename, self.masks[index])
196
197 matched = []
198 for key, (frame, image_name, image) in phase_by_key.items():
199 mask_entry = masks_by_key.get(key)
200 if mask_entry is None:
201 self._record_processing_error(
202 "phase_mask_matching",
203 ValueError(f"No mask matches phase field '{image_name}'."),
204 image_name=image_name,
205 )
206 continue
207 mask_name, mask = mask_entry
208 if image.ndim != 2 or mask.ndim != 2:
209 self._record_processing_error(
210 "phase_mask_matching",
211 ValueError(
212 f"Phase and mask images must be 2D; got phase shape "
213 f"{image.shape} and mask '{mask_name}' shape {mask.shape}."
214 ),
215 image_name=image_name,
216 )
217 continue
218 if image.shape != mask.shape:
219 self._record_processing_error(
220 "phase_mask_matching",
221 ValueError(
222 f"Phase shape {image.shape} does not match mask "
223 f"'{mask_name}' shape {mask.shape}."
224 ),
225 image_name=image_name,
226 )
227 continue
228 matched.append((frame, image_name, image, mask))
229
230 for key, (mask_name, _) in masks_by_key.items():
231 if key not in phase_by_key:
232 self._record_processing_error(
233 "phase_mask_matching",
234 ValueError(f"No phase image matches mask '{mask_name}'."),
235 image_name=mask_name,
236 )
237 if not matched:
238 raise ValueError("No phase images have matching masks and dimensions.")
239 return matched
240 def _normalize_channel_list(self, channel_list, argument_name="channel_list"):
241 if channel_list is None:
242 raise ValueError(
243 f"No {argument_name} provided. It should be of format ['channelname1', 'channelname2']."
244 )
245 if isinstance(channel_list, str):
246 raise TypeError(
247 f"{argument_name} must be an iterable of channel names, not a single string."
248 )
249 normalized_channels = list(channel_list)
250 if not normalized_channels:
251 raise ValueError(f"{argument_name} must contain at least one channel name.")
252 return normalized_channels
253
254 def _iter_processing_targets(self, object_list, phase_channel):
255 if isinstance(object_list, Image):
256 return [object_list]
257 if object_list is None:
258 self.create_image_objects(phase_channel=phase_channel)
259 return self.image_objects
260 if isinstance(object_list, (list, tuple)):
261 return list(object_list)
262 raise TypeError(
263 "object_list must be None, a single Image instance, or a list/tuple of Image instances."
264 )
265
266 def _get_mesh_dataframe_lookup(self):
267 """Build an image-name lookup from the current mesh dataframe.
268
269 Do not retain this lookup across calls: callers can replace or mutate
270 ``mesh_df_collection``, and a cached lookup could then return stale
271 meshes for newly loaded data.
272 """
273 if not isinstance(self.mesh_df_collection, pd.DataFrame) or self.mesh_df_collection.empty:
274 return None
275 return {
276 image_name: group.reset_index(drop=True)
277 for image_name, group in self.mesh_df_collection.groupby(
278 "image_name", sort=False
279 )
280 }
281
282
283 def load_masks(self):
284 """
285 Read masks from the ``/masks`` subfolder.
286
287 Mutates ``self.masks`` and ``self.mask_filenames``.
288
289 """
290 self._require_image_folder_path()
291 self.masks, self.mask_filenames = u.read_tiff_folder(
292 self.image_folder_path + "/masks"
293 )
294 self._record_stage_success("load_masks")
295
296 def load_phase_images(self, phase_channel=""):
297 """
298 This method loads phase contrast images from the specified image folder path, based on the characters before the file-extension,
299 and stores them in the 'self.images' array.
300
301 Parameters
302 ----------
303 phase_channel : str, optional
304 Characters before the file-extension (e.g., "c1", "phase") to look for when using this function. (default is an empty string)
305
306 Mutates ``self.images``, ``self.image_filenames`` and ``self.paths``.
307
308 """
309 self._require_image_folder_path()
310 self.phase_channel = phase_channel or ""
311 try:
312 self.images, self.image_filenames, self.paths = u.read_tiff_folder(
313 self.image_folder_path, phase_channel, include_paths=True
314 )
315 except Exception as e:
316 raise type(e)(
317 f"Failed to load phase contrast images from '{self.image_folder_path}' "
318 f"with phase_channel='{phase_channel}': {e}"
319 ) from e
320 self._record_stage_success("load_phase_images")
321
322 def load_channel_images(self, channel_list):
323 """
324 This method loads channel images from the image folder path based on the specified characters before the file-extension and stores them in
325 the 'self.channel_images' dictionary, where the channel name is used as the key.
326
327 Parameters
328 ----------
329 channel_list : List of str
330 Characters before the extension (e.g., ["c2", "c3"]) to look for when using this function.
331
332 Mutates ``self.channel_images``.
333
334 """
335 self._require_image_folder_path()
336 self.channel_list = self._normalize_channel_list(channel_list)
337 for channel in self.channel_list:
338 # A failed read must not leave arrays from an earlier folder/load in place.
339 self.channel_images.pop(channel, None)
340 self.channel_image_filenames.pop(channel, None)
341 self.channel_images_by_field.pop(channel, None)
342 try:
343 images, filenames = u.read_tiff_folder(
344 self.image_folder_path, suffix=channel
345 )
346 except Exception as e:
347 self._record_processing_error(
348 "load_channel_images", e, channel=channel
349 )
350 continue
351
352 self.channel_images[channel] = list(images)
353 self.channel_image_filenames[channel] = list(filenames)
354 keyed_images = {}
355 for filename, image in zip(filenames, images):
356 key = u.image_field_key(filename, suffix=channel)
357 if key in keyed_images:
358 self._record_processing_error(
359 "load_channel_images",
360 ValueError(f"Duplicate channel field identity '{key}'."),
361 image_name=filename,
362 channel=channel,
363 )
364 continue
365 keyed_images[key] = image
366 self.channel_images_by_field[channel] = keyed_images
367 self._record_stage_success("load_channel_images")
368
369 def create_image_objects(self, phase_channel=""):
370 """
371 Create ``Image`` objects from the loaded masks and phase images.
372
373 Invalid collection-level setup raises immediately. Failures while creating
374 a single image object are logged and skipped so the rest of the batch can continue.
375 """
376 if self.segmentation_failed:
377 raise RuntimeError(
378 "Segmentation failed in this collection; refusing to load masks "
379 "from disk because they may belong to an earlier run."
380 )
381 if self.masks is None:
382 self.load_masks()
383 if self.images is None or self.image_filenames is None:
384 self.load_phase_images(phase_channel)
385
386 matched_images = self._validate_phase_image_state()
387
388 # Feature tables and merged output belong to the previous image/cell set.
389 self.feature_dataframes = {}
390 self.merged_features = None
391 self.image_objects = []
392 mesh_lookup = self._get_mesh_dataframe_lookup()
393
394 for i, image_name, image, mask in matched_images:
395 try:
396 mesh_df = mesh_lookup.get(image_name) if mesh_lookup is not None else None
397
398 img_obj = self.create_image_object(
399 image, image_name, i, mask, mesh_df, self.px
400 )
401 self.image_objects.append(img_obj)
402 except Exception as e:
403 bactoscoop_logger.debug(traceback.format_exc())
404 self._record_processing_error(
405 "create_image_objects", e, image_name=image_name
406 )
407 self._record_stage_success("create_image_objects")
408
409 def add_channels(self, img_objects, channel_list, load_data=True):
410 """
411 This method loads channel images based on the provided channel list and adds them to the individual image objects.
412
413 Parameters
414 ----------
415 img_objects : List
416 List of image objects.
417 channel_list : List
418 List of channel names.
419
420 """
421 channel_list = self._normalize_channel_list(channel_list)
422 if img_objects is None or len(img_objects) == 0:
423 raise ValueError("img_objects must contain at least one image object.")
424
425 # Load requested channels while keeping field identity beside each array.
426 if load_data:
427 self.load_channel_images(channel_list)
428
429 expected_fields = {
430 u.image_field_key(obj.image_name, suffix=self.phase_channel)
431 for obj in img_objects
432 }
433 for channel in channel_list:
434 for extra_field in self.channel_images_by_field.get(channel, {}):
435 if extra_field not in expected_fields:
436 self._record_processing_error(
437 "match_channel_image",
438 ValueError(
439 f"No phase image object matches {channel} field '{extra_field}'."
440 ),
441 image_name=extra_field,
442 channel=channel,
443 )
444
445 for img_obj in img_objects:
446 channel_single_image_dict = dict(getattr(img_obj, "channels", None) or {})
447 field_key = u.image_field_key(
448 img_obj.image_name, suffix=self.phase_channel
449 )
450 for channel in channel_list:
451 # Never retain an earlier image or derived interpolation for a
452 # requested channel if its current file is missing or invalid.
453 channel_single_image_dict.pop(channel, None)
454 for cache_name in ("bg_channels", "chann_interp2d"):
455 cache = getattr(img_obj, cache_name, None)
456 if isinstance(cache, dict):
457 cache.pop(channel, None)
458 channel_fields = self.channel_images_by_field.get(channel, {})
459 channel_image = channel_fields.get(field_key)
460 if channel_image is None:
461 self._record_processing_error(
462 "match_channel_image",
463 ValueError(
464 f"No {channel} image matches field '{img_obj.image_name}'."
465 ),
466 image_name=img_obj.image_name,
467 channel=channel,
468 )
469 continue
470 if channel_image.ndim != 2 or img_obj.image.ndim != 2:
471 self._record_processing_error(
472 "match_channel_image",
473 ValueError(
474 f"{channel} and phase images must be 2D; got channel "
475 f"shape {channel_image.shape} and phase shape {img_obj.image.shape}."
476 ),
477 image_name=img_obj.image_name,
478 channel=channel,
479 )
480 continue
481 if channel_image.shape != img_obj.image.shape:
482 self._record_processing_error(
483 "match_channel_image",
484 ValueError(
485 f"{channel} image shape {channel_image.shape} does not "
486 f"match phase shape {img_obj.image.shape}."
487 ),
488 image_name=img_obj.image_name,
489 channel=channel,
490 )
491 continue
492 channel_single_image_dict[channel] = channel_image
493 img_obj.channels = channel_single_image_dict or None
494 self._record_stage_success("add_channels")
495
496 def get_mesh_dataframe(self, image_name):
497 """
498 This method returns the mesh dataframe associated with the specified image name.
499
500 Parameters
501 ----------
502 image_name : str
503 The name or identifier of the image.
504
505 """
506
507 mesh_lookup = self._get_mesh_dataframe_lookup()
508 if mesh_lookup is None:
509 return None
510 return mesh_lookup.get(image_name)
511
512 def create_image_object(self, image, image_name, index, mask, mesh_df, px):
513 """
514 This method creates a single Image object using the provided image, image name, index, mask, mesh dataframe, and pixel size (px).
515 If mesh data is present, it also creates associated Cell objects.
516
517 Parameters
518 ----------
519 image : phase contrast image
520 The phase contrast image for the Image object.
521 image_name : str
522 The name or identifier of the image.
523 index : int
524 The index or position of the image.
525 mask : mask or None
526 The mask or region of interest associated with the image.
527 mesh_df : pd.DataFrame or None
528 The mesh dataframe associated with the image, if available, or None.
529 px : float
530 The pixel size in micrometers.
531 """
532
533 img_obj = Image(image, image_name, index, mask, mesh_df, px=px)
534
535 if mesh_df is not None:
536 img_obj.create_cell_object(verbose=False)
537 return img_obj
538
539 # Methods for batch processing of images ----------------------------------------------
540 def segment_images(self, mask_thresh, minsize, n, model_name="bact_phase_omni"):
541 self.segmentation_failed = False
542 self.masks = None
543 self.mask_filenames = None
544 try:
545 from .omni import Omnipose
546 import torch
547
548 omni = Omnipose(self.images, self.paths)
549 omni.load_models(model_name)
550
551 omni.compiled_process(n, mask_thresh, minsize)
552 torch.cuda.empty_cache()
553
554 del omni
555 gc.collect()
556 self._record_stage_success("segment_images")
557 return True
558 except Exception as e:
559 self.segmentation_failed = True
560 bactoscoop_logger.debug(traceback.format_exc())
561 self._record_processing_error("segment_images", e)
562 return False
563
564 def batch_detect_objects(
565 self,
566 channels=None,
567 reset_channels=True,
568 align=False,
569 smoothing=0.1,
570 log_sigma=3,
571 kernel_width=4,
572 min_overlap_ratio=0.01,
573 max_external_ratio=0.1,
574 ):
575 """
576 Detect objects within the contour of the cell.
577
578 Parameters
579 ----------
580 channels : List of strings, optional
581 The channel names of the channels to perform object detection on.
582
583 log_sigma : int, optional
584 The sigma parameter of the Laplacian of Gaussian filter used in object detection. (default is 3)
585
586 kernel_width : int, optional
587 The kernel width of the kernel used for dilating the object masks.
588 Decreasing results in smaller masks, while increasing results in larger masks. (default is 4)
589
590 min_overlap_ratio : float, optional
591 Minimum overlap between the object and cell required for the object to be kept. (default is 0.01)
592
593 max_external_ratio : float, optional
594 The maximum ratio an object is allowed to lie outside the cell contour. (default is 0.1)
595
596 Returns
597 -------
598 pd.DataFrame
599 A DataFrame containing cell_id, object_contours, and frame information.
600
601 """
602 self._require_loaded_images()
603 channels = self._normalize_channel_list(channels, argument_name="channels")
604
605 if reset_channels:
606 self.channel_images = {}
607 self.channel_image_filenames = {}
608 self.channel_images_by_field = {}
609 for image in self.image_objects:
610 image.channels = None
611 for cache_name in ("bg_channels", "chann_interp2d"):
612 cache = getattr(image, cache_name, None)
613 if isinstance(cache, dict):
614 cache.clear()
615
616 channels_to_load = [
617 channel for channel in channels
618 if channel not in self.channel_images_by_field
619 ]
620 if channels_to_load:
621 self.load_channel_images(channels_to_load)
622 self.add_channels(self.image_objects, channels, load_data=False)
623
624 bactoscoop_logger.info("Detecting objects within cell ...")
625
626 dfs = []
627 for image in tqdm_if_verbose(self.image_objects):
628 for cell in getattr(image, "cells", []):
629 for channel in channels:
630 cell.object_meshdata.pop(channel, None)
631 if image.channels is None or any(
632 channel not in image.channels for channel in channels
633 ):
634 # The missing field has already been recorded by add_channels.
635 continue
636 try:
637 detection_df = image.object_detection(
638 channels,
639 smoothing,
640 align,
641 log_sigma,
642 kernel_width,
643 min_overlap_ratio,
644 max_external_ratio,
645 )
646 except Exception as e:
647 bactoscoop_logger.debug(traceback.format_exc())
648 self._record_processing_error(
649 "object_detection", e, image_name=image.image_name
650 )
651 continue
652 dfs.append(detection_df)
653 cell_errors = getattr(image, "object_detection_errors", [])
654 for cell_error in cell_errors:
655 self._record_cell_error(
656 "cell_object_detection",
657 RuntimeError(cell_error["error"]),
658 image_name=image.image_name,
659 channel=cell_error.get("channel"),
660 cell_id=cell_error.get("cell_id"),
661 error_type=cell_error.get("error_type"),
662 )
663 cell_count = len(getattr(image, "cells", []))
664 failed_attempts = sum(
665 not error.get("recovered", False) for error in cell_errors
666 )
667 if cell_count and failed_attempts >= cell_count * len(channels):
668 self._record_processing_error(
669 "object_detection",
670 ValueError(
671 f"All cell object-detection attempts failed for '{image.image_name}'."
672 ),
673 image_name=image.image_name,
674 )
675
676 self.object_detection_df = (
677 pd.concat(dfs, axis=0, ignore_index=True) if dfs else pd.DataFrame()
678 )
679
680 if not self.object_detection_df.empty and "channel" in self.object_detection_df:
681 bactoscoop_logger.info(self.object_detection_df["channel"].value_counts())
682 else:
683 bactoscoop_logger.info("No objects were detected.")
684
685 self._record_stage_success("batch_detect_objects")
686 return self.object_detection_df
687
688 def batch_process_mesh(
689 self,
690 pkl_name=None,
691 object_list=None,
692 save_data=True,
693 phase_channel="",
694 join_thresh=4,
695 split_thresh=0.35,
696 CD_width=False,
697 smoothing=0.1,
698 neighbor_filter_max_neighbors=None,
699 neighbor_filter_connectivity=1,
700 max_daughter_cell_mesh_rows=800,
701 ):
702 """
703 Process cellular meshes from segmented masks and store the data in Cell objects.
704
705 After segmentation, this method can process the resulting masks and create cellular meshes from them.
706 It first joins cells with a pole-pole distance (px) below the 'join_thresh'.
707 Subsequently, meshes of all cells are created and cells can be split again if the constriction degree exceeds the 'split_thresh'.
708 The data is then stored in Cell objects and can be exported in a pickle file.
709
710 Parameters
711 ----------
712 pkl_name : str, optional
713 Name of the pickle file to save the mesh data. If None, the file will be saved as 'folder_name_meshdata.pkl'.
714
715 object_list : List, optional
716 Specify a list of Image objects to be processed. If 'object_list' is None, all Image instances in 'image_objects' are processed.
717
718 phase_channel : str, optional
719 Characters before the file-extension (.tif, .tiff) that the algorithm looks for when using this function (e.g., "c1", "phase").
720
721 join_thresh : int, optional
722 The pole-pole distance threshold for joining cells. Cells with a distance below this threshold will have their masks joined. (default is 4)
723
724 split_thresh : float, optional
725 The constriction degree threshold for splitting cells. Cells with a constriction degree exceeding this threshold will be split. (default is 0.35)
726
727 max_daughter_cell_mesh_rows : int, optional
728 Maximum rebuilt daughter mesh rows (default 800, minimum 4).
729 Oversized resampling is rejected before allocation; both final
730 daughters must have 4..limit rows. This is separate from the feature
731 extraction max_mesh_size parameter and does not limit unsplit parents.
732
733 """
734 max_daughter_cell_mesh_rows = u._validate_max_daughter_cell_mesh_rows(
735 max_daughter_cell_mesh_rows
736 )
737 self.mesh_df_collection = pd.DataFrame()
738 self.mesh_prefilter_stats = pd.DataFrame()
739 object_list = self._iter_processing_targets(object_list, phase_channel)
740 mesh_frames = []
741 prefilter_stats = []
742
743 for img_obj in object_list:
744 bactoscoop_logger.info(f"PROCESSING IMAGE: {img_obj.image_name}")
745 try:
746 img_obj.join_split_pipeline(
747 join_thresh,
748 split_thresh,
749 CD_width,
750 smoothing,
751 neighbor_filter_max_neighbors=neighbor_filter_max_neighbors,
752 neighbor_filter_connectivity=neighbor_filter_connectivity,
753 max_daughter_cell_mesh_rows=max_daughter_cell_mesh_rows,
754 )
755 mesh_df = getattr(img_obj, "processed_mesh_dataframe", None)
756 if mesh_df is None:
757 raise ValueError(
758 f"Image '{img_obj.image_name}' did not produce a processed mesh dataframe."
759 )
760 mesh_frames.append(mesh_df)
761 if getattr(img_obj, "prefilter_summary", None) is not None:
762 prefilter_stats.append(
763 {
764 "image_name": img_obj.image_name,
765 **img_obj.prefilter_summary,
766 }
767 )
768 img_obj.create_cell_object(verbose=False)
769 except Exception as e:
770 bactoscoop_logger.debug(traceback.format_exc())
771 self._record_processing_error(
772 "mesh_processing", e, image_name=img_obj.image_name
773 )
774
775 self.mesh_df_collection = (
776 pd.concat(mesh_frames, ignore_index=True) if mesh_frames else pd.DataFrame()
777 )
778 self.mesh_prefilter_stats = (
779 pd.DataFrame(prefilter_stats) if prefilter_stats else pd.DataFrame()
780 )
781
782 if save_data:
783 if pkl_name is not None:
784 self._to_pickle(pkl_name, self.mesh_df_collection)
785
786 self._to_pickle(f"{self.name}_meshdata.pkl", self.mesh_df_collection)
787
788 self._record_stage_success("batch_process_mesh")
789
790
791 def batch_load_mesh(self, pkl_name, pkl_path=None, phase_channel=""):
792 """
793 Load mesh data from a pickle-file and create image objects from the data.
794
795 Parameters
796 ----------
797 pkl_name : str
798 Name of the pickle file where the mesh data is stored (e.g., 'Ecoli_meshdata.pkl').
799
800 pkl_path : str, optional
801 Path to the file where the mesh data is stored. If not provided, the method will look in the image folder path.
802
803 phase_channel : str, optional
804 Characters before the file-extension (.tif, .tiff) that the algorithm looks for when using this function (e.g., "c1", "phase").
805
806 """
807 self.mesh_df_collection = None
808 if pkl_path is None:
809 pkl_path = self.image_folder_path
810
811 full_path = os.path.join(pkl_path, pkl_name)
812 if not os.path.isfile(full_path):
813 raise FileNotFoundError(f"Pickle file not found: {full_path}")
814
815 if full_path.lower().endswith(".parquet"):
816 mesh_df = pd.read_parquet(full_path, engine="pyarrow")
817 for column in ("contour", "mesh", "midline"):
818 if column not in mesh_df:
819 raise ValueError(f"Mesh Parquet file is missing '{column}'.")
820 mesh_df[column] = mesh_df[column].map(
821 lambda value: np.asarray(value.tolist(), dtype=float)
822 )
823 self.mesh_df_collection = mesh_df
824 else:
825 with open(full_path, "rb") as f:
826 self.mesh_df_collection = pickle.load(f)
827
828 bactoscoop_logger.info("Meshes loaded from file: %s", pkl_name)
829 self.create_image_objects(phase_channel=phase_channel)
830
831 def batch_mask2mesh(
832 self,
833 pkl_name=None,
834 object_list=None,
835 save_data=True,
836 phase_channel="",
837 smoothing=0.1,
838 ):
839 """
840 Create meshes from masks without the joining or splitting as in the batch_process_mesh method.
841
842 Parameters
843 ----------
844 pkl_name : str, optional
845 Name of the pickle file to save the mesh data. If None, the file will be saved as 'folder_name_meshdata.pkl'.
846 object_list : List, optional
847 Specify a list of Image objects to be processed. If 'object_list' is None, all Image instances in 'image_objects' are processed.
848 phase_channel : str, optional
849 Characters before the file-extension (.tif, .tiff) that the algorithm looks for when using this function (e.g., "c1", "phase").
850 """
851 self.mesh_df_collection = pd.DataFrame()
852 mesh_frames = []
853
854 if isinstance(object_list, Image):
855 print("\n")
856 bactoscoop_logger.info(f"PROCESSING IMAGE: {object_list.image_name}")
857
858 object_list.mask2mesh(smoothing)
859 mesh_frames.append(getattr(object_list, "mesh_dataframe"))
860 object_list.create_cell_object(verbose=False)
861 else:
862
863 if object_list is None:
864 self.create_image_objects(phase_channel=phase_channel)
865 object_list = self.image_objects
866
867 for img_obj in object_list:
868
869 bactoscoop_logger.info(f"PROCESSING IMAGE: {img_obj.image_name}")
870
871 img_obj.mask2mesh()
872 mesh_frames.append(getattr(img_obj, "mesh_dataframe"))
873
874 img_obj.create_cell_object(verbose=False)
875
876 self.mesh_df_collection = (
877 pd.concat(mesh_frames, ignore_index=True) if mesh_frames else pd.DataFrame()
878 )
879
880 if save_data:
881 if pkl_name is not None:
882 self._to_pickle(pkl_name, self.mesh_df_collection)
883
884 self._to_pickle(
885 "{}_meshdata.pkl".format(self.name), self.mesh_df_collection
886 )
887
888 def batch_shift_correction(
889 self,
890 shifted_channel,
891 reset_channels=False,
892 log_sigma=1.5,
893 kernel_width=3,
894 min_overlap_ratio=0.1,
895 max_external_ratio=1,
896 phase_log_sigma=0.5,
897 phase_closing_level=2,
898 signal_closing_level=12,
899 max_shift_correction=15,
900 ):
901 self._require_loaded_images()
902 shifted_channel = self._normalize_channel_list(
903 shifted_channel, argument_name="shifted_channel"
904 )
905
906 if reset_channels:
907 self.channel_images = {}
908 self.channel_image_filenames = {}
909 self.channel_images_by_field = {}
910 for image in self.image_objects:
911 image.channels = None
912 for cache_name in ("bg_channels", "chann_interp2d"):
913 cache = getattr(image, cache_name, None)
914 if isinstance(cache, dict):
915 cache.clear()
916
917 channels_to_load = [
918 channel for channel in shifted_channel
919 if channel not in self.channel_images_by_field
920 ]
921 if channels_to_load:
922 self.load_channel_images(channels_to_load)
923 self.add_channels(self.image_objects, shifted_channel, load_data=False)
924
925 success_percentage_list = []
926 bactoscoop_logger.info("Aligning contour, mesh and midline to channel image ...")
927 for image_obj in tqdm_if_verbose(self.image_objects):
928 if image_obj.channels is None or any(
929 channel not in image_obj.channels for channel in shifted_channel
930 ):
931 # Missing channels were recorded by add_channels.
932 continue
933 try:
934 success_percentage = image_obj.shift_correction(
935 shifted_channel,
936 log_sigma,
937 kernel_width,
938 min_overlap_ratio,
939 max_external_ratio,
940 phase_log_sigma,
941 phase_closing_level,
942 signal_closing_level,
943 max_shift_correction,
944 )
945 except Exception as e:
946 bactoscoop_logger.debug(traceback.format_exc())
947 self._record_processing_error(
948 "shift_correction", e, image_name=image_obj.image_name
949 )
950 continue
951 success_percentage_list.append(success_percentage)
952 cell_errors = getattr(image_obj, "shift_errors", [])
953 for cell_error in cell_errors:
954 self._record_cell_error(
955 "cell_shift_correction",
956 RuntimeError(cell_error["error"]),
957 image_name=image_obj.image_name,
958 channel=cell_error.get("channel"),
959 cell_id=cell_error.get("cell_id"),
960 error_type=cell_error.get("error_type"),
961 )
962 cell_count = len(getattr(image_obj, "cells", []))
963 if cell_count and len(cell_errors) >= cell_count * len(shifted_channel):
964 self._record_processing_error(
965 "shift_correction",
966 ValueError(
967 f"All cell shift-correction attempts failed for '{image_obj.image_name}'."
968 ),
969 image_name=image_obj.image_name,
970 )
971 if not success_percentage_list:
972 bactoscoop_logger.warning("No shift-correction results were produced.")
973 return 0.0
974
975 average_success_percentage = sum(success_percentage_list) / len(
976 success_percentage_list
977 )
978
979 bactoscoop_logger.info(f"Shift succes rate: {average_success_percentage}")
980 return average_success_percentage
981
982 def batch_calculate_features(
983 self,
984 channel_method_pairs,
985 all_data=True,
986 reset=True,
987 use_shifted_contours=False,
988 shift_signal=False,
989 max_mesh_size=600,
990 retain_contour_on_object_mesh_failure_channels=None,
991 ):
992 """
993 Calculate features of all image objects for specified channels and methods.
994
995 This method calculates features for all image objects based on the specified channel-method pairs. It takes a list of tuples, where each tuple contains channel names and the feature calculation method. The resulting feature dataframes are stored in a dictionary.
996
997 Parameters
998 ----------
999 channel_method_pairs : List[Tuple[List[str], str]]
1000 A list of tuples where each tuple contains the channel names and feature calculation method.
1001 Example of channel_method_pairs: [(['c1'], 'profiling'), (['c2', 'c3'], 'objects'), ([None], 'svm')].
1002 With the example pairs, this method will analyze c1 using the profiling method,
1003 then c2 and c3 using the objects method, and finally the phase contrast channel using the svm method.
1004
1005 add_profiling_data : bool, optional
1006 Whether to include profiling data in the feature calculation. (default is True)
1007
1008 reset : bool, optional
1009 If True, the dictionary storing the dataframes will be reset. If False, dataframes are added to the existing dictionary. (default is True)
1010 retain_contour_on_object_mesh_failure_channels : list[str] or None, optional
1011 Channels whose objects features retain contour-based measurements
1012 when any detected object mesh is invalid. Other channels omit that
1013 cell's objects feature row. None is equivalent to an empty list.
1014 This option does not affect other feature methods.
1015
1016 Returns
1017 -------
1018 Dict
1019 A dictionary containing dataframes with calculated features.
1020
1021 """
1022 self._require_loaded_images()
1023 if channel_method_pairs is None:
1024 raise ValueError("channel_method_pairs must be provided.")
1025 if retain_contour_on_object_mesh_failure_channels is None:
1026 retained_channels = frozenset()
1027 elif isinstance(retain_contour_on_object_mesh_failure_channels, str):
1028 raise TypeError(
1029 "retain_contour_on_object_mesh_failure_channels must be a list "
1030 "of channel names, not a string."
1031 )
1032 else:
1033 retained_channels = frozenset(
1034 retain_contour_on_object_mesh_failure_channels
1035 )
1036 if not all(isinstance(channel, str) for channel in retained_channels):
1037 raise TypeError(
1038 "retain_contour_on_object_mesh_failure_channels must contain "
1039 "only channel names."
1040 )
1041
1042 self.merged_features = None
1043 if reset:
1044 self.feature_dataframes = {}
1045
1046 for channels, method in channel_method_pairs:
1047 normalized_channels = (
1048 [None]
1049 if channels == [None]
1050 else self._normalize_channel_list(channels, argument_name="channels")
1051 )
1052 for channel in normalized_channels:
1053 key = f"{method}_{channel}_features"
1054 self.feature_dataframes.pop(key, None)
1055 bactoscoop_logger.info(
1056 f"PROCESSING CHANNEL: {channel}, method: {method}"
1057 )
1058 feature_dfs = []
1059
1060 for img_obj in tqdm_if_verbose(self.image_objects):
1061 if channel is not None and (
1062 img_obj.channels is None or channel not in img_obj.channels
1063 ):
1064 self._record_processing_error(
1065 "image_feature_calculation",
1066 ValueError(
1067 f"Channel {channel} is not loaded for image '{img_obj.image_name}'."
1068 ),
1069 image_name=img_obj.image_name,
1070 channel=channel,
1071 )
1072 continue
1073 try:
1074 result_df = img_obj.calculate_features(
1075 method,
1076 channel,
1077 all_data,
1078 use_shifted_contours,
1079 shift_signal,
1080 max_mesh_size,
1081 retained_channels,
1082 )
1083 except Exception as e:
1084 bactoscoop_logger.debug(traceback.format_exc())
1085 self._record_processing_error(
1086 "image_feature_calculation",
1087 e,
1088 image_name=img_obj.image_name,
1089 channel=channel,
1090 )
1091 continue
1092 if result_df is not None:
1093 feature_dfs.append(result_df)
1094 for feature_error in getattr(img_obj, "feature_errors", []):
1095 self._record_cell_error(
1096 "cell_feature_calculation",
1097 RuntimeError(feature_error["error"]),
1098 image_name=img_obj.image_name,
1099 channel=channel,
1100 cell_id=feature_error.get("cell_id"),
1101 error_type=feature_error.get("error_type"),
1102 )
1103
1104 if feature_dfs:
1105 concatenated_df = pd.concat(feature_dfs, axis=0)
1106 concatenated_df.reset_index(drop=True, inplace=True)
1107 self.feature_dataframes[key] = concatenated_df
1108 has_rows = not concatenated_df.empty
1109 else:
1110 has_rows = False
1111 if not has_rows:
1112 self._record_processing_error(
1113 "batch_calculate_features",
1114 ValueError(
1115 f"No feature rows were produced for channel {channel}, method {method}."
1116 ),
1117 channel=channel,
1118 )
1119
1120 self._record_stage_success("batch_calculate_features")
1121 return self.feature_dataframes
1122
1123 def batch_calculate_signal_correlation_features(
1124 self, df, channels, feature_method_tuples
1125 ):
1126 from .signalcorrelation import SignalCorrelation
1127
1128 # Generate all unique pairs of channels
1129 channel_pairs = combinations(channels, 2)
1130
1131 for channel1, channel2 in channel_pairs:
1132 for features, method_names in feature_method_tuples:
1133 bactoscoop_logger.info(
1134 f"Processing channel {channel1} vs {channel2} and features {features} from {self.image_folder_path}:"
1135 )
1136 for feature in tqdm_if_verbose(features):
1137 prepared_feature = SignalCorrelation.prepare_feature_pair(
1138 df, channel1, channel2, feature
1139 )
1140
1141 for method_name in method_names:
1142 sc = SignalCorrelation(
1143 df,
1144 channel1,
1145 channel2,
1146 feature,
1147 method_name,
1148 prepared_feature=prepared_feature,
1149 )
1150 sc.calculate()
1151 del sc
1152 return df
1153
1154 def _apply_curation_labels(self):
1155 """Apply curation labels by stable image identity, with frame fallback."""
1156 image_objects_by_name = {}
1157 image_objects_by_frame = {}
1158 for image in self.image_objects:
1159 image_objects_by_name.setdefault(image.image_name, []).append(image)
1160 image_objects_by_frame.setdefault(image.frame, []).append(image)
1161
1162 for _, row in self.curated_df.iterrows():
1163 image_name = row.get("image_name")
1164 cell_id = row["cell_id"]
1165 label = row["label"]
1166 image = None
1167
1168 if pd.notna(image_name):
1169 matches = image_objects_by_name.get(image_name, [])
1170 if len(matches) == 1:
1171 image = matches[0]
1172 else:
1173 reason = (
1174 f"Image name '{image_name}' is not unique."
1175 if len(matches) > 1
1176 else f"No image object matches image name '{image_name}'."
1177 )
1178 self._record_processing_error(
1179 "curation_mapping",
1180 ValueError(reason),
1181 image_name=image_name,
1182 cell_id=cell_id,
1183 )
1184 continue
1185
1186 if image is None:
1187 frame = row.get("frame")
1188 matches = image_objects_by_frame.get(frame, [])
1189 if len(matches) == 1:
1190 image = matches[0]
1191 else:
1192 self._record_processing_error(
1193 "curation_mapping",
1194 ValueError(
1195 f"Could not uniquely map curated cell {cell_id} to image "
1196 f"(image_name={image_name!r}, frame={frame!r})."
1197 ),
1198 image_name=image_name if pd.notna(image_name) else None,
1199 cell_id=cell_id,
1200 )
1201 continue
1202
1203 frame = row.get("frame")
1204 if pd.notna(frame) and image.frame != frame:
1205 self._record_processing_error(
1206 "curation_mapping",
1207 ValueError(
1208 f"Curated cell {cell_id} names image '{image.image_name}' "
1209 f"but has frame {frame}; expected {image.frame}."
1210 ),
1211 image_name=image.image_name,
1212 cell_id=cell_id,
1213 )
1214 continue
1215
1216 cells = [cell for cell in image.cells if cell.cell_id == cell_id]
1217 if len(cells) != 1:
1218 reason = (
1219 f"Cell ID {cell_id} was not found in image '{image.image_name}'."
1220 if not cells
1221 else f"Cell ID {cell_id} is not unique in image '{image.image_name}'."
1222 )
1223 self._record_processing_error(
1224 "curation_mapping",
1225 ValueError(reason),
1226 image_name=image.image_name,
1227 cell_id=cell_id,
1228 )
1229 continue
1230 if label == 0:
1231 image.cells.remove(cells[0])
1232
1233 self._record_stage_success("curate_dataset")
1234
1235 def curate_dataset(
1236 self, path_to_model, cols=4, control=False, save_curated_data=True
1237 ):
1238 """
1239 Curate the dataset based on a trained support vector machine model.
1240
1241 This method calculates SVM features and uses a trained SVM model to classify cells based on these features.
1242 It curates the dataset by discarding cells that are classified as label 0 (non-interesting).
1243 Optionally, it can include positive and negative control cells.
1244
1245 Parameters:
1246 path_to_model (str): Path to the trained SVM model.
1247
1248 cols (int): Retained for backward compatibility; currently unused.
1249
1250 control (bool): Whether to include positive and negative control cells.
1251
1252 save_curated_data (bool): Whether to write the curated mesh pickle.
1253 The in-memory mesh dataframe is updated either way.
1254
1255 Returns:
1256 None
1257 """
1258 self.batch_calculate_features(
1259 channel_method_pairs=[([None], "svm")], all_data=True, reset=True, max_mesh_size=800
1260 )
1261
1262 from .curation import Curation
1263
1264 cur = Curation(self.feature_dataframes["svm_None_features"])
1265
1266 self.curated_df = cur.compiled_curation(path_to_model, cols)
1267
1268 p1, p0 = cur.get_label_proportions()
1269 bactoscoop_logger.info(
1270 f"\nProportion of label 1: {p1}\nProportion of label 0: {p0}"
1271 )
1272
1273 if control:
1274 import bactoscoop.plot as plotting
1275
1276 self.pos, self.neg = cur.get_control(5, 5)
1277 if self.pos and self.neg is not None:
1278 bactoscoop_logger.info(
1279 f"\nCell IDs and Frames for positive control are {self.pos}\nFrame and Cell IDs for negative control are {self.neg}"
1280 )
1281 plotting.plot_svm_controls(
1282 self.pos, self.image_objects, message="Positive"
1283 )
1284 plotting.plot_svm_controls(
1285 self.neg, self.image_objects, message="Negative"
1286 )
1287
1288 self._apply_curation_labels()
1289
1290
1291 # The in-memory mesh population follows the curated cells even when
1292 # the caller is only testing and does not want a file overwritten.
1293 self.add_meshdata_to_dataframe()
1294 if save_curated_data:
1295 self._to_pickle(
1296 "{}_curated_meshdata.pkl".format(self.name), self.mesh_df_collection
1297 )
1298
1299 del cur
1300
1301 def add_meshdata_to_dataframe(self):
1302 data = []
1303
1304 # Iterate over each image object
1305 for image in self.image_objects:
1306 # Iterate over each cell in the current image
1307 for cell in image.cells:
1308 # Extract the required attributes from the cell and image
1309 cell_id = cell.cell_id
1310 contour = cell.contour
1311 midline = cell.midline
1312 mesh = cell.mesh
1313 frame = image.frame
1314 image_name = image.image_name
1315
1316 # Append the data as a dictionary to the list
1317 data.append(
1318 {
1319 "image_name": image_name,
1320 "frame": frame,
1321 "cell_id": cell_id,
1322 "contour": contour,
1323 "midline": midline,
1324 "mesh": mesh,
1325 }
1326 )
1327
1328 # Create DataFrame from the collected data
1329 self.mesh_df_collection = pd.DataFrame(data)
1330
1331 def export_curated_masks(self, output_subfolder="curated_masks", overwrite=True):
1332 """Write one label mask per image from the SVM-curated cell contours.
1333
1334 This method is intended to be run after :meth:`curate_dataset`. It never
1335 changes the original Omnipose masks in ``masks``. Instead, it writes new
1336 masks to ``output_subfolder`` and records how every curated label overlaps
1337 the original labels, so the output can be used directly by BactoFeatures.
1338
1339 Parameters
1340 ----------
1341 output_subfolder : str, optional
1342 Folder below ``image_folder_path`` for the exported label TIFFs.
1343 overwrite : bool, optional
1344 If ``False``, fail before replacing an existing curated-mask TIFF.
1345
1346 Returns
1347 -------
1348 tuple[list[str], pandas.DataFrame]
1349 Paths to the exported masks and a provenance table with one row per
1350 curated cell.
1351 """
1352 self._require_image_folder_path()
1353 if not self.image_objects:
1354 raise ValueError(
1355 "No image objects are available. Load curated mesh data or run "
1356 "curate_dataset() before exporting curated masks."
1357 )
1358
1359 output_dir = os.path.join(self.image_folder_path, output_subfolder)
1360 os.makedirs(output_dir, exist_ok=True)
1361 mask_paths = []
1362 provenance_rows = []
1363
1364 for image in self.image_objects:
1365 if image.mask is None:
1366 raise ValueError(f"Original mask is not loaded for {image.image_name}.")
1367
1368 max_label = len(image.cells)
1369 output_dtype = np.uint16 if max_label <= np.iinfo(np.uint16).max else np.uint32
1370 curated_mask = np.zeros(image.mask.shape, dtype=output_dtype)
1371 stem = os.path.splitext(image.image_name)[0]
1372 output_path = os.path.join(output_dir, f"{stem}_curated_masks.tif")
1373 if os.path.exists(output_path) and not overwrite:
1374 raise FileExistsError(
1375 f"Curated mask already exists: {output_path}. Set overwrite=True to replace it."
1376 )
1377
1378 for curated_label, cell in enumerate(image.cells, start=1):
1379 contour = np.asarray(cell.contour, dtype=float)
1380 if contour.ndim != 2 or contour.shape[1] != 2 or len(contour) < 3:
1381 bactoscoop_logger.warning(
1382 "Skipping curated cell %s in %s: invalid contour.",
1383 cell.cell_id,
1384 image.image_name,
1385 )
1386 continue
1387
1388 cell_mask = np.zeros(image.mask.shape, dtype=np.uint8)
1389 # Cell contours are (row, column); OpenCV expects (x, y).
1390 polygon = np.rint(contour[:, ::-1]).astype(np.int32).reshape(-1, 1, 2)
1391 cv.fillPoly(cell_mask, [polygon], color=1)
1392 cell_pixels = cell_mask.astype(bool)
1393 if not cell_pixels.any():
1394 bactoscoop_logger.warning(
1395 "Skipping curated cell %s in %s: contour has no in-image pixels.",
1396 cell.cell_id,
1397 image.image_name,
1398 )
1399 continue
1400
1401 curated_mask[cell_pixels] = curated_label
1402 source_labels = np.unique(image.mask[cell_pixels])
1403 source_labels = source_labels[source_labels != 0].astype(int).tolist()
1404 rows, columns = np.nonzero(cell_pixels)
1405 provenance_rows.append(
1406 {
1407 "image_name": image.image_name,
1408 "frame": image.frame,
1409 "curated_label": curated_label,
1410 "bactoscoop_cell_id": cell.cell_id,
1411 "centroid_x_px": float(columns.mean()),
1412 "centroid_y_px": float(rows.mean()),
1413 "area_px": int(cell_pixels.sum()),
1414 "original_mask_path": os.path.join(
1415 self.image_folder_path, "masks", f"{stem}_cp_masks.tif"
1416 ),
1417 "original_mask_labels": ";".join(map(str, source_labels)),
1418 "curated_mask_path": output_path,
1419 }
1420 )
1421
1422 tifffile.imwrite(output_path, curated_mask)
1423 mask_paths.append(output_path)
1424
1425 provenance = pd.DataFrame(provenance_rows)
1426 provenance_path = os.path.join(output_dir, "curated_mask_provenance.csv")
1427 provenance.to_csv(provenance_path, index=False)
1428 bactoscoop_logger.info(
1429 "Exported %s curated masks and %s curated cells to %s",
1430 len(mask_paths),
1431 len(provenance),
1432 output_dir,
1433 )
1434 return mask_paths, provenance
1435
1436 def merge_dataframes(self, include_metadata_tag=False, discard_morphological_nan=False):
1437 """
1438 Merge feature dataframes into a single dataframe.
1439
1440 Merges multiple feature dataframes into a single dataframe, ensuring that columns are properly renamed
1441 to identify the channel and removes duplicates based on specific columns.
1442
1443 Args:
1444 include_metadata_tag (bool): Whether to rename 'image_name', 'cell_id', and 'frame' to 'Metadata_*'.
1445 discard_nan (bool): Whether to discard rows with NaN in 'cell_area' and 'cell_length'.
1446
1447 Returns:
1448 pd.DataFrame: The merged feature dataframe.
1449 """
1450 if not self.feature_dataframes:
1451 raise ValueError("No feature dataframes are available to merge.")
1452
1453 merged_dataframes = []
1454
1455 for key, df in self.feature_dataframes.items():
1456 # Keys are "method_channel_features"; channel names may contain underscores.
1457 if not key.endswith("_features"):
1458 raise ValueError(f"Invalid feature dataframe key: {key!r}")
1459 method_and_channel = key[: -len("_features")]
1460 method, separator, channel = method_and_channel.partition("_")
1461 if not separator or not method or not channel:
1462 raise ValueError(f"Invalid feature dataframe key: {key!r}")
1463
1464 # Set a prefix to identify the channel in column names if the channel is not None
1465 if channel != "None":
1466 df = df.rename(
1467 columns={
1468 col: f"{channel}_{col}"
1469 for col in df.columns
1470 if col not in ["image_name", "cell_id", "frame"]
1471 }
1472 )
1473
1474 # Append the modified dataframe to the list
1475 merged_dataframes.append(df)
1476
1477 # Concatenate the dataframes vertically
1478 merged_data = pd.concat(merged_dataframes)
1479
1480 # Merge the dataframes based on 'image_name', 'cell_id', and 'frame'
1481 result = merged_data.pivot_table(
1482 index=["image_name", "cell_id", "frame"], aggfunc="first"
1483 ).reset_index()
1484
1485 # Discard rows with NaN in 'cell_area' and 'cell_length' if discard_nan is True
1486 if discard_morphological_nan:
1487 result = result.dropna(subset=["cell_area", "cell_length"])
1488
1489 if include_metadata_tag:
1490 # Rename 'image_name', 'cell_id', and 'frame' to 'Metadata_*'
1491 result = result.rename(
1492 columns={
1493 "image_name": "Metadata_image_name",
1494 "cell_id": "Metadata_cell_id",
1495 "frame": "Metadata_frame"
1496 }
1497 )
1498
1499 # Reset the index to make it look like the final result
1500 self.merged_features = result.reset_index(drop=True)
1501
1502 return self.merged_features
1503
1504 def dataframe_to_pkl(self, pkl_name=None):
1505 """Save merged features in the image folder and return the pickle path.
1506
1507 A suffixless ``pkl_name`` is a legacy tag appended to the dataset name;
1508 a name ending in ``.pkl`` is used as the exact filename.
1509 """
1510 if self.merged_features is None:
1511 raise ValueError("Merge feature dataframes before saving them.")
1512 if pkl_name is None:
1513 filename = f"{self.name}_features.pkl"
1514 elif str(pkl_name).endswith(".pkl"):
1515 filename = str(pkl_name)
1516 else:
1517 filename = f"{self.name}_{pkl_name}.pkl"
1518 self._to_pickle(filename, self.merged_features)
1519 return os.path.join(self.image_folder_path, filename)
1520
1521 def meshdata_to_parquet(self, parquet_name=None):
1522 """Save the current mesh dataframe as reloadable Parquet in the image folder.
1523
1524 Geometry arrays become nested numeric lists in Parquet. Reload with
1525 ``batch_load_mesh("name.parquet", phase_channel=...)``.
1526 Requires the optional ``pyarrow`` package.
1527 """
1528 if self.mesh_df_collection is None or self.mesh_df_collection.empty:
1529 raise ValueError("No mesh data is available to save.")
1530 name = parquet_name or f"{self.name}_meshdata.parquet"
1531 self._to_parquet(name, self.mesh_df_collection, mesh_data=True)
1532
1533 def dataframe_to_parquet(self, parquet_name=None):
1534 """Save merged features as Parquet in the image folder (requires pyarrow)."""
1535 if self.merged_features is None:
1536 raise ValueError("Merge feature dataframes before saving them.")
1537 name = parquet_name or f"{self.name}_features.parquet"
1538 self._to_parquet(name, self.merged_features)
1539
1540 def _atomic_write(self, filename, writer):
1541 self._require_image_folder_path()
1542 if os.path.basename(filename) != filename or filename in {"", ".", ".."}:
1543 raise ValueError("Output filename must be a name within the image folder.")
1544 file_path = os.path.join(self.image_folder_path, filename)
1545 descriptor, temporary_path = tempfile.mkstemp(
1546 prefix=f".{filename}.", suffix=".tmp", dir=self.image_folder_path
1547 )
1548 os.close(descriptor)
1549 try:
1550 writer(temporary_path)
1551 with open(temporary_path, "rb+") as temporary_file:
1552 os.fsync(temporary_file.fileno())
1553 os.replace(temporary_path, file_path)
1554 finally:
1555 if os.path.exists(temporary_path):
1556 os.unlink(temporary_path)
1557
1558 def _to_parquet(self, filename, data, mesh_data=False):
1559 if not filename.lower().endswith(".parquet"):
1560 raise ValueError("Parquet filename must end in .parquet")
1561 output = data.copy()
1562 if mesh_data:
1563 for column in ("contour", "mesh", "midline"):
1564 if column not in output:
1565 raise ValueError(f"Mesh dataframe is missing '{column}'.")
1566 output[column] = output[column].map(
1567 lambda value: np.asarray(value).tolist()
1568 )
1569 self._atomic_write(
1570 filename,
1571 lambda path: output.to_parquet(
1572 path, engine="pyarrow", compression="zstd", index=False
1573 ),
1574 )
1575
1576 def _to_pickle(self, pkl_name, data):
1577 """
1578 Save data to a pkl file.
1579
1580 """
1581 self._atomic_write(
1582 pkl_name,
1583 lambda path: self._write_pickle_file(path, data),
1584 )
1585
1586 @staticmethod
1587 def _write_pickle_file(path, data):
1588 with open(path, "wb") as file:
1589 pickle.dump(data, file, protocol=pickle.HIGHEST_PROTOCOL)