Bases: TextValidator[float]
Validate caption-image agreement with SIGLIP 2.
Load the SIGLIP model and processor.
Source code in modosaic/text/validators/impl/siglip_2.py
| def __init__(self) -> None:
"""Load the SIGLIP model and processor."""
self.device, self.dtype = DeviceService.get_device_type()
self.model = self._load_model()
self.model.to(self.device)
self.processor = self._load_processor()
|
validate
validate(record, generated)
Score a generated caption against the source image.
Source code in modosaic/text/validators/impl/siglip_2.py
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 | @override
@torch.no_grad()
def validate(self, record: ImageRecord, generated: str) -> float:
"""Score a generated caption against the source image."""
logger.debug(f"Validating text-image embedding for record sample {record.sample_id}")
image = ImageService.bytes_to_pil(record.image_bytes)
text_description = [f"{VALIDATION_ENRICHMENT} {generated}"]
inputs = self.processor(
text=text_description,
images=image,
padding="max_length",
max_num_patches=256,
return_tensors="pt"
).to(self.model.device)
outputs = self.model(**inputs)
logits_per_image = outputs.logits_per_image
probs = torch.sigmoid(logits_per_image)
probability = probs[0][0].item()
logger.debug(f"Text-image probability for record sample {record.sample_id}: {probability}")
return probability
|