Merge pull request #71 from macaodha/enhancement/logs-to-file

Add CLI log file output

docs/source/reference/cli/index.md

@@ -17,6 +17,8 @@ for the full option list.
 ## Notes
 
 - Global CLI options are documented in {doc}`base`.
+- Use `--log-file path/to/cli.log` to save CLI logs to a file while still
+  showing them in the terminal.
 - Paths with spaces should be wrapped in quotes.
 - Input audio is expected to be mono.
 - `process` uses the optional `--detection-threshold` override.

docs/source/tutorials/run-inference-on-folder.md

@@ -97,6 +97,23 @@ What this does:
 - runs the model on each recording,
 - saves the results in `path/to/outputs`.
 
+```{eval-rst}
+.. admonition:: Save CLI logs to a file
+   :collapsible: closed
+   :class: tip dropdown
+
+   If you want to keep a copy of the CLI logs, add ``--log-file`` before the
+   subcommand:
+
+   .. code-block:: bash
+
+      batdetect2 --log-file path/to/cli.log process directory \
+        path/to/audio \
+        path/to/outputs
+
+   This writes the same CLI logs to ``path/to/cli.log`` and to the terminal.
+```
+
 You do not need to choose a model for this first run.
 If you do nothing, BatDetect2 uses the built-in default UK model.
 

src/batdetect2/cli/base.py

@@ -1,5 +1,7 @@
 """BatDetect2 command line interface."""
 
+from pathlib import Path
+
 import click
 
 from batdetect2.cli.ascii import BATDETECT_ASCII_ART
@@ -22,8 +24,17 @@ BatDetect2
     count=True,
     help="Increase verbosity. -v for INFO, -vv for DEBUG.",
 )
+@click.option(
+    "--log-file",
+    type=click.Path(path_type=Path),
+    help="Write CLI logs to a file in addition to the terminal.",
+)
 @click.pass_context
-def cli(ctx: click.Context, verbose: int = 0):
+def cli(
+    ctx: click.Context,
+    verbose: int = 0,
+    log_file: Path | None = None,
+) -> None:
     """Run the BatDetect2 CLI.
 
     Use subcommands to run processing, training, evaluation, and dataset
@@ -37,4 +48,4 @@ def cli(ctx: click.Context, verbose: int = 0):
 
     from batdetect2.logging import enable_logging
 
-    enable_logging(verbose)
+    enable_logging(verbose, log_file=log_file)

src/batdetect2/evaluate/plots/detection.py

@@ -21,7 +21,7 @@ from batdetect2.core import ImportConfig, Registry, add_import_config
 from batdetect2.evaluate.metrics.common import compute_precision_recall
 from batdetect2.evaluate.metrics.detection import ClipEval
 from batdetect2.evaluate.plots.base import BasePlot, BasePlotConfig
-from batdetect2.plotting.detections import plot_clip_detections
+from batdetect2.plotting.detections import plot_clip_evaluation
 from batdetect2.plotting.metrics import plot_pr_curve, plot_roc_curve
 from batdetect2.preprocess import PreprocessingConfig, build_preprocessor
 from batdetect2.preprocess.types import PreprocessorProtocol
@@ -276,7 +276,7 @@ class ExampleDetectionPlot(BasePlot):
             fig = self.create_figure()
             ax = fig.subplots()
 
-            plot_clip_detections(
+            plot_clip_evaluation(
                 clip_eval,
                 ax=ax,
                 audio_loader=self.audio_loader,

src/batdetect2/inference/clips.py

@@ -2,7 +2,9 @@ from typing import List, Sequence
 from uuid import uuid5
 
 import numpy as np
+from loguru import logger
 from soundevent import data
+from soundfile import LibsndfileError
 
 
 def get_clips_from_files(
@@ -16,7 +18,15 @@ def get_clips_from_files(
     clips: List[data.Clip] = []
 
     for path in paths:
-        recording = data.Recording.from_file(path, compute_hash=compute_hash)
+        try:
+            recording = data.Recording.from_file(
+                path,
+                compute_hash=compute_hash,
+            )
+        except LibsndfileError as e:
+            logger.warning(f"Skipping unreadable audio file {path}: {e}")
+            continue
+
         clips.extend(
             get_recording_clips(
                 recording,

src/batdetect2/logging.py

@@ -50,7 +50,7 @@ __all__ = [
 ]
 
 
-def enable_logging(level: int):
+def enable_logging(level: int, log_file: Path | None = None) -> None:
     logger.remove()
 
     if level == 0:
@@ -61,6 +61,11 @@ def enable_logging(level: int):
         log_level = "DEBUG"
 
     logger.add(sys.stderr, level=log_level)
+
+    if log_file is not None:
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+        logger.add(log_file, level=log_level)
+
     logger.enable("batdetect2")
 
 

src/batdetect2/outputs/formats/batdetect2.py

@@ -1,4 +1,5 @@
 import json
+from collections import defaultdict
 from pathlib import Path
 from typing import List, Literal, Sequence, TypedDict, cast
 
@@ -93,8 +94,11 @@ class BatDetect2Formatter(OutputFormatterProtocol[FileAnnotation]):
     def format(
         self, predictions: Sequence[ClipDetections]
     ) -> List[FileAnnotation]:
+        merged_predictions = merge_clip_detections(predictions)
+
         return [
-            self.format_prediction(prediction) for prediction in predictions
+            self.format_prediction(prediction)
+            for prediction in merged_predictions
         ]
 
     def save(
@@ -349,3 +353,48 @@ class BatDetect2Formatter(OutputFormatterProtocol[FileAnnotation]):
             preserve_audio_tree=config.preserve_audio_tree,
             include_file_path=config.include_file_path,
         )
+
+
+def merge_clip_detections(
+    predictions: Sequence[ClipDetections],
+) -> List[ClipDetections]:
+    """Merge clip predictions into one recording-level prediction.
+
+    This intentionally discards the original clip boundaries because the
+    legacy BatDetect2 file format only stores recording-level detections.
+    """
+    rec_to_clips = defaultdict(list)
+    rec_mapping = {}
+
+    for prediction in predictions:
+        recording = prediction.clip.recording
+        key = recording.path
+        rec_to_clips[key].append(prediction)
+        rec_mapping[key] = recording
+
+    merged_predictions = []
+    for rec_path, clips in rec_to_clips.items():
+        recording = rec_mapping[rec_path]
+        merged_predictions.append(
+            ClipDetections(
+                clip=data.Clip(
+                    recording=recording,
+                    start_time=0,
+                    end_time=recording.duration,
+                ),
+                detections=sorted(
+                    [
+                        detection
+                        for clip_detections in clips
+                        for detection in clip_detections.detections
+                    ],
+                    key=lambda detection: (
+                        detection.detection_score,
+                        *compute_bounds(detection.geometry),
+                    ),
+                    reverse=True,
+                ),
+            )
+        )
+
+    return merged_predictions

src/batdetect2/plotting/detections.py

@@ -1,4 +1,6 @@
+import numpy as np
 from matplotlib import axes, patches
+from soundevent.geometry import compute_bounds
 from soundevent.plot import plot_geometry
 
 from batdetect2.evaluate.metrics.detection import ClipEval
@@ -8,13 +10,111 @@ from batdetect2.plotting.clips import (
     plot_clip,
 )
 from batdetect2.plotting.common import create_ax
+from batdetect2.postprocess import ClipDetections, Detection
 
 __all__ = [
-    "plot_clip_detections",
+    "plot_clip_evaluation",
+    "plot_detection",
 ]
 
 
-def plot_clip_detections(
+def plot_detection(
+    detection: Detection,
+    figsize: tuple[int, int] = (10, 10),
+    ax: axes.Axes | None = None,
+    fill: bool = False,
+    linewidth: float = 1.0,
+    linestyle: str = "--",
+    color: str = "red",
+    show_class: bool = True,
+    class_names: list[str] | None = None,
+    fontsize: float | str = "small",
+):
+    ax = create_ax(figsize=figsize, ax=ax)
+
+    plot_geometry(
+        detection.geometry,
+        ax=ax,
+        add_points=False,
+        facecolor="none" if not fill else color,
+        alpha=detection.detection_score,
+        linewidth=linewidth,
+        linestyle=linestyle,
+        color=color,
+    )
+
+    if not show_class:
+        return ax
+
+    start_time, low_freq, _, _ = compute_bounds(detection.geometry)
+
+    top_class = np.argmax(detection.class_scores)
+    score = detection.class_scores[top_class]
+
+    if class_names is not None:
+        class_name = class_names[top_class]
+    else:
+        class_name = f"class {top_class}"
+
+    ax.text(
+        start_time,
+        low_freq,
+        f"{class_name}={score:.2f}",
+        va="top",
+        ha="left",
+        color=color,
+        fontsize=fontsize,
+        alpha=detection.detection_score,
+    )
+    return ax
+
+
+def plot_clip_detection(
+    clip_detections: ClipDetections,
+    figsize: tuple[int, int] = (10, 10),
+    ax: axes.Axes | None = None,
+    audio_loader: AudioLoader | None = None,
+    preprocessor: PreprocessorProtocol | None = None,
+    threshold: float | None = None,
+    spec_cmap: str = "gray",
+    fill: bool = False,
+    linewidth: float = 1.0,
+    linestyle: str = "--",
+    color: str = "red",
+    show_class: bool = True,
+    class_names: list[str] | None = None,
+    fontsize: float | str = "small",
+):
+    ax = create_ax(figsize=figsize, ax=ax)
+
+    plot_clip(
+        clip_detections.clip,
+        audio_loader=audio_loader,
+        preprocessor=preprocessor,
+        ax=ax,
+        spec_cmap=spec_cmap,
+    )
+
+    for detection in clip_detections.detections:
+        if threshold and detection.detection_score < threshold:
+            continue
+
+        ax = plot_detection(
+            detection,
+            ax=ax,
+            class_names=class_names,
+            fontsize=fontsize,
+            fill=fill,
+            linewidth=linewidth,
+            linestyle=linestyle,
+            color=color,
+            show_class=show_class,
+        )
+
+    return ax
+
+
+def plot_clip_evaluation(
     clip_eval: ClipEval,
     figsize: tuple[int, int] = (10, 10),
     ax: axes.Axes | None = None,

src/batdetect2/utils/visualize.py

@@ -52,9 +52,9 @@ class InteractivePlotter:
         self.spec_slices = spec_slices
         self.call_info = call_info
         # _, self.labels = np.unique([cc['class'] for cc in call_info], return_inverse=True)
-        self.labels = np.zeros(len(call_info), dtype=np.int)
+        self.labels = np.zeros(len(call_info), dtype=int)
         self.annotated = np.zeros(
-            self.labels.shape[0], dtype=np.int
+            self.labels.shape[0], dtype=int
         )  # can populate this with 1's where we have labels
         self.labels_cols = [
             colors[self.labels[ii]] for ii in range(len(self.labels))

tests/test_cli/test_base.py

@@ -1,9 +1,13 @@
 """Behavior-focused tests for top-level CLI command discovery."""
 
+from pathlib import Path
+
 from click.testing import CliRunner
 
 from batdetect2.cli import cli
 
+BASE_DIR = Path(__file__).parent.parent.parent
+
 
 def test_cli_base_help_lists_main_commands() -> None:
     """User story: discover available workflows from top-level help."""
@@ -11,8 +15,43 @@ def test_cli_base_help_lists_main_commands() -> None:
     result = CliRunner().invoke(cli, ["--help"])
 
     assert result.exit_code == 0
+    assert "--log-file" in result.output
     assert "process" in result.output
     assert "train" in result.output
     assert "evaluate" in result.output
     assert "data" in result.output
     assert "detect" in result.output
+
+
+def test_cli_writes_logs_to_file_and_terminal(
+    tmp_path: Path,
+    tiny_checkpoint_path: Path,
+) -> None:
+    """User story: save CLI logs to a file while keeping terminal output."""
+
+    output_dir = tmp_path / "eval_out"
+    log_file = tmp_path / "logs" / "cli.log"
+
+    result = CliRunner().invoke(
+        cli,
+        [
+            "--log-file",
+            str(log_file),
+            "-v",
+            "evaluate",
+            str(BASE_DIR / "example_data" / "dataset.yaml"),
+            "--model",
+            str(tiny_checkpoint_path),
+            "--base-dir",
+            str(BASE_DIR),
+            "--workers",
+            "0",
+            "--output-dir",
+            str(output_dir),
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert "Initiating evaluation process..." in result.output
+    assert log_file.exists()
+    assert "Initiating evaluation process..." in log_file.read_text()

tests/test_cli/test_process.py

@@ -0,0 +1,131 @@
+import json
+import shutil
+from collections import Counter
+from pathlib import Path
+
+from click.testing import CliRunner
+from soundevent.geometry import compute_bounds
+
+from batdetect2 import BatDetect2API
+from batdetect2.cli import cli
+
+
+def test_cli_process_directory_merges_clip_outputs_per_recording(
+    tmp_path: Path,
+    contrib_dir: Path,
+) -> None:
+    recording_path = contrib_dir / "jeff37" / "0166_20240531_223911.wav"
+
+    source_folder = tmp_path / "audio"
+    source_folder.mkdir()
+    shutil.copy2(
+        recording_path,
+        source_folder / "example_audio.wav",
+    )
+
+    destination_folder = tmp_path / "results"
+    destination_folder.mkdir()
+
+    api = BatDetect2API.from_checkpoint()
+
+    api_outputs = api.process_directory(
+        source_folder,
+        detection_threshold=0.3,
+    )
+
+    # Get all detections regardless of clip
+    detections = [
+        detection
+        for clip_detections in api_outputs
+        for detection in clip_detections.detections
+    ]
+
+    result = CliRunner().invoke(
+        cli,
+        args=[
+            "process",
+            "directory",
+            str(source_folder),
+            str(destination_folder),
+            "--detection-threshold",
+            "0.3",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert destination_folder.exists()
+
+    output_json = destination_folder / "example_audio.wav.json"
+    assert output_json.exists()
+
+    saved_detections = json.loads(output_json.read_text())
+
+    expected_annotations = Counter(
+        (
+            round(float(start_time), 4),
+            round(float(end_time), 4),
+            int(low_freq),
+            int(high_freq),
+            round(float(detection.class_scores.max()), 3),
+            round(float(detection.detection_score), 3),
+        )
+        for detection in detections
+        for start_time, low_freq, end_time, high_freq in [
+            compute_bounds(detection.geometry)
+        ]
+    )
+
+    actual_annotations = Counter(
+        (
+            annotation["start_time"],
+            annotation["end_time"],
+            annotation["low_freq"],
+            annotation["high_freq"],
+            annotation["class_prob"],
+            annotation["det_prob"],
+        )
+        for annotation in saved_detections["annotation"]
+    )
+
+    assert actual_annotations == expected_annotations
+
+
+def test_cli_process_directory_skips_corrupted_files(
+    tmp_path: Path,
+    contrib_dir: Path,
+) -> None:
+    recording_path = contrib_dir / "jeff37" / "0166_20240531_223911.wav"
+
+    source_folder = tmp_path / "audio"
+    source_folder.mkdir()
+    shutil.copy2(
+        recording_path,
+        source_folder / "example_audio.wav",
+    )
+
+    corrupted_file = source_folder / "corrupted.wav"
+    corrupted_file.write_text("corrupted")
+
+    destination_folder = tmp_path / "results"
+    destination_folder.mkdir()
+
+    result = CliRunner().invoke(
+        cli,
+        args=[
+            "process",
+            "directory",
+            str(source_folder),
+            str(destination_folder),
+            "--detection-threshold",
+            "0.3",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert destination_folder.exists()
+
+    output_json = destination_folder / "example_audio.wav.json"
+    assert output_json.exists()
+
+    corrupted_file_json = destination_folder / "corrupted.wav.json"
+    assert not corrupted_file_json.exists()

tests/test_inference/test_batch.py

@@ -0,0 +1,55 @@
+from pathlib import Path
+
+import pytest
+from soundevent import data
+
+from batdetect2.inference.batch import run_batch_inference
+from batdetect2.targets import build_roi_mapping, build_targets
+from batdetect2.train import load_model_from_checkpoint
+from tests.utils import assert_clip_detections_equal
+
+pytestmark = pytest.mark.slow
+
+
+def test_run_batch_inference_matches_single_clip_inference(
+    contrib_dir: Path,
+) -> None:
+    recording = data.Recording.from_file(
+        contrib_dir / "jeff37" / "0166_20240531_223911.wav"
+    )
+    clips = [
+        data.Clip(recording=recording, start_time=start, end_time=start + 1.0)
+        for start in (0.0, 1.0, 2.0)
+    ]
+    model, configs = load_model_from_checkpoint()
+    targets = build_targets(configs.targets)
+    roi_mapper = build_roi_mapping(configs.targets.roi)
+
+    batched_predictions = run_batch_inference(
+        model,
+        clips,
+        targets=targets,
+        roi_mapper=roi_mapper,
+        batch_size=3,
+        num_workers=0,
+    )
+    single_predictions = [
+        run_batch_inference(
+            model,
+            [clip],
+            targets=targets,
+            roi_mapper=roi_mapper,
+            batch_size=1,
+            num_workers=0,
+        )[0]
+        for clip in clips
+    ]
+
+    assert len(batched_predictions) == len(single_predictions)
+
+    for batched, single in zip(
+        batched_predictions,
+        single_predictions,
+        strict=True,
+    ):
+        assert_clip_detections_equal(batched, single)

tests/test_utils/test_visualize.py

@@ -0,0 +1,22 @@
+import numpy as np
+
+from batdetect2.utils.visualize import InteractivePlotter
+
+
+def test_interactive_plotter_init_builds_integer_label_arrays():
+    feats_ds = np.zeros((2, 2))
+    spec_slices = [np.zeros((4, 6)), np.zeros((4, 8))]
+    call_info = [{"class": "a"}, {"class": "b"}]
+
+    plotter = InteractivePlotter(
+        feats_ds=feats_ds,
+        feats=feats_ds,
+        spec_slices=spec_slices,
+        call_info=call_info,
+        freq_lims=[0, 1],
+        allow_training=False,
+    )
+
+    assert plotter.labels.shape == (2,)
+    assert np.issubdtype(plotter.labels.dtype, np.integer)
+    assert np.issubdtype(plotter.annotated.dtype, np.integer)

tests/utils.py

@@ -0,0 +1,57 @@
+import numpy as np
+from soundevent.geometry import compute_bounds
+
+from batdetect2.postprocess.types import ClipDetections
+
+
+def assert_clip_detections_equal(
+    detections: ClipDetections,
+    other: ClipDetections,
+) -> None:
+    """Assert two clip-detection objects are numerically equivalent."""
+    assert detections.clip.recording.path == other.clip.recording.path
+    assert detections.clip.start_time == other.clip.start_time
+    assert detections.clip.end_time == other.clip.end_time
+    assert len(detections.detections) == len(other.detections)
+
+    sorted_detections = sorted(
+        detections.detections,
+        key=lambda det: (
+            compute_bounds(det.geometry)[0],
+            compute_bounds(det.geometry)[1],
+        ),
+    )
+
+    sorted_other = sorted(
+        other.detections,
+        key=lambda det: (
+            compute_bounds(det.geometry)[0],
+            compute_bounds(det.geometry)[1],
+        ),
+    )
+
+    for det, other_det in zip(
+        sorted_detections,
+        sorted_other,
+        strict=True,
+    ):
+        np.testing.assert_allclose(
+            np.array(compute_bounds(det.geometry)),
+            np.array(compute_bounds(other_det.geometry)),
+            atol=2e-2,
+        )
+        assert np.isclose(
+            det.detection_score,
+            other_det.detection_score,
+            atol=1e-6,
+        )
+        np.testing.assert_allclose(
+            det.class_scores,
+            other_det.class_scores,
+            atol=1e-6,
+        )
+        np.testing.assert_allclose(
+            det.features,
+            other_det.features,
+            atol=2e-6,
+        )