modosaic.providers.image_dataset¶

modosaic.providers.image_dataset ¶

ImageDataset ¶

ImageDataset(adapter)

Bases: Iterable[ImageRecord]

Iterable image dataset facade over a concrete backend adapter.

Initialize the dataset.

Parameters:

Name	Type	Description	Default
`adapter`	`DatasetAdapter`	Backend adapter that yields `ImageRecord` objects.	required

Source code in modosaic/providers/image_dataset.py

def __init__(self, adapter: DatasetAdapter):
    """Initialize the dataset.

    Args:
        adapter: Backend adapter that yields `ImageRecord` objects.
    """
    self.adapter = adapter

iter ¶

__iter__()

Yield image records from the configured adapter.

Source code in modosaic/providers/image_dataset.py

def __iter__(self) -> Iterator[ImageRecord]:
    """Yield image records from the configured adapter."""
    yield from self.adapter.iter_samples()

from_local_folder `classmethod` ¶

from_local_folder(root, recursive=True, extensions=None)

Create a dataset from a local image folder.

Parameters:

Name	Type	Description	Default
`root`	`str \| Path`	Folder containing images.	required
`recursive`	`bool`	Whether to scan subdirectories.	`True`
`extensions`	`Iterable[str] \| None`	Optional accepted image extensions.	`None`

Returns:

Type	Description
`ImageDataset`	Dataset backed by `LocalFolderAdapter`.

Source code in modosaic/providers/image_dataset.py

@classmethod
def from_local_folder(
        cls,
        root: str | Path,
        recursive: bool = True,
        extensions: Iterable[str] | None = None,
) -> "ImageDataset":
    """Create a dataset from a local image folder.

    Args:
        root: Folder containing images.
        recursive: Whether to scan subdirectories.
        extensions: Optional accepted image extensions.

    Returns:
        Dataset backed by `LocalFolderAdapter`.
    """
    return cls(LocalFolderAdapter(root, recursive=recursive, extensions=extensions))

from_parquet `classmethod` ¶

from_parquet(parquet_path, image_column='image', id_column=None, extension_column=None, metadata_columns=None, batch_size=512)

Create a dataset from a parquet file or directory.

Parameters:

Name	Type	Description	Default
`parquet_path`	`str \| Path`	Parquet file or directory containing parquet files.	required
`image_column`	`str`	Column or nested path containing image bytes or paths.	`'image'`
`id_column`	`str \| None`	Optional sample identifier column.	`None`
`extension_column`	`str \| None`	Optional image extension column.	`None`
`metadata_columns`	`Iterable[str] \| None`	Optional columns copied into record metadata.	`None`
`batch_size`	`int`	Number of rows read from parquet at a time.	`512`

Returns:

Type	Description
`ImageDataset`	Dataset backed by `ParquetAdapter`.

Source code in modosaic/providers/image_dataset.py

@classmethod
def from_parquet(
        cls,
        parquet_path: str | Path,
        image_column: str = "image",
        id_column: str | None = None,
        extension_column: str | None = None,
        metadata_columns: Iterable[str] | None = None,
        batch_size: int = 512,
) -> "ImageDataset":
    """Create a dataset from a parquet file or directory.

    Args:
        parquet_path: Parquet file or directory containing parquet files.
        image_column: Column or nested path containing image bytes or paths.
        id_column: Optional sample identifier column.
        extension_column: Optional image extension column.
        metadata_columns: Optional columns copied into record metadata.
        batch_size: Number of rows read from parquet at a time.

    Returns:
        Dataset backed by `ParquetAdapter`.
    """
    return cls(
        ParquetAdapter(
            parquet_path,
            image_column=image_column,
            id_column=id_column,
            extension_column=extension_column,
            metadata_columns=metadata_columns,
            batch_size=batch_size,
        )
    )

save_to_folder ¶

save_to_folder(output_dir, overwrite=False, naming=None)

Persist every dataset image to a local folder.

Parameters:

Name	Type	Description	Default
`output_dir`	`str \| Path`	Destination directory.	required
`overwrite`	`bool`	Whether to overwrite existing files. When false, duplicate filenames receive numeric suffixes.	`False`
`naming`	`Callable[[ImageRecord], str] \| None`	Optional callback that returns a filename for each sample.	`None`

Returns:

Type	Description
`list[Path]`	Paths written to disk.

Source code in modosaic/providers/image_dataset.py

def save_to_folder(
        self,
        output_dir: str | Path,
        overwrite: bool = False,
        naming: Callable[[ImageRecord], str] | None = None,
) -> list[Path]:
    """Persist every dataset image to a local folder.

    Args:
        output_dir: Destination directory.
        overwrite: Whether to overwrite existing files. When false,
            duplicate filenames receive numeric suffixes.
        naming: Optional callback that returns a filename for each sample.

    Returns:
        Paths written to disk.
    """
    output_path = Path(output_dir)
    output_path.mkdir(parents=True, exist_ok=True)

    saved_paths: list[Path] = []
    for sample in self:
        filename = naming(sample) if naming else sample.filename
        filename_path = Path(filename)
        stem = ExtensionService.sanitize_fragment(filename_path.stem)
        suffix = ExtensionService.normalize_extension(filename_path.suffix or sample.extension)

        destination = output_path / f"{stem}{suffix}"
        if not overwrite:
            count = 1
            while destination.exists():
                destination = output_path / f"{stem}_{count}{suffix}"
                count += 1

        destination.write_bytes(sample.image_bytes)
        saved_paths.append(destination)
        logger.debug(f"Saved image dataset sample {sample.sample_id} to {destination}")
    return saved_paths

modosaic.providers.image_dataset¶