fix: bound model input images below 512 KiB

This commit is contained in:
Soulter
2026-09-17 20:24:21 +08:00
parent 2704b4c0d3
commit f3ae8cda18
13 changed files with 408 additions and 641 deletions
+1 -23
View File
@@ -198,10 +198,8 @@ DEFAULT_CONFIG = {
"cua_local": CUA_DEFAULT_CONFIG["local"],
"cua_api_key": CUA_DEFAULT_CONFIG["api_key"],
},
"image_compress_enabled": True,
"image_compress_options": {
"max_size": 1280,
"quality": 95,
},
},
"agent_runner": {
@@ -3025,9 +3023,6 @@ CONFIG_METADATA_2 = {
"provider_settings": {
"type": "object",
"items": {
"image_compress_enabled": {
"type": "bool",
},
"enable": {
"type": "bool",
},
@@ -4209,29 +4204,12 @@ CONFIG_METADATA_3 = {
"type": "string",
"hint": "如果唤醒前缀为 /, 额外聊天唤醒前缀为 chat,则需要 /chat 才会触发 LLM 请求",
},
"provider_settings.image_compress_enabled": {
"description": "启用图片压缩",
"type": "bool",
"hint": "默认开启。发送给多模态模型前按需压缩转换图片:合规的 JPEG/PNG 原样发送,动图生成拼图预览。",
},
"provider_settings.image_compress_options.max_size": {
"description": "最大边长",
"type": "int",
"hint": "压缩后图片的最长边,单位为像素,超出则按比例缩放。CUA 沙箱下输入图片不缩放,大图可能超出服务商上传限制。",
"condition": {
"provider_settings.image_compress_enabled": True,
},
"hint": "发送给模型的图片最长边,单位为像素。图片始终自动处理,单张文件严格小于 512 KiB;必要时会进一步缩小。原始附件及其路径保持不变。",
"slider": {"min": 256, "max": 4096, "step": 64},
},
"provider_settings.image_compress_options.quality": {
"description": "压缩质量",
"type": "int",
"hint": "JPEG 输出质量,范围为 1-100。值越高,画质越好,文件也越大。",
"condition": {
"provider_settings.image_compress_enabled": True,
},
"slider": {"min": 1, "max": 100, "step": 1},
},
"provider_settings.prompt_prefix": {
"description": "用户提示词",
"type": "string",
@@ -2,15 +2,10 @@
from pathlib import Path
from astrbot.core import logger
from astrbot.core.agent.message import ImageURLPart, TextPart
from astrbot.core.platform.astr_message_event import AstrMessageEvent
from astrbot.core.provider.entities import ProviderRequest
from astrbot.core.utils.media_utils import (
MediaResolver,
is_recoverable_image_error,
prepare_model_image,
)
from astrbot.core.utils.media_utils import prepare_model_image
from astrbot.core.utils.string_utils import normalize_and_dedupe_strings
@@ -18,26 +13,20 @@ async def prepare_request_images(
req: ProviderRequest,
event: AstrMessageEvent,
*,
enabled: bool,
max_size: int,
quality: int,
output_dir: Path,
prepared: dict[str, str | None],
quote_image_ref: str | None = None,
montage_max_size: int | None = None,
) -> None:
"""Replace current images on a working request and track their owned files.
Args:
req: Working request; shared lists and image blocks are copied on write.
event: Owner of downloaded source files and prepared working files.
enabled: Whether the current pipeline profile enables model image preparation.
max_size: Normalized longest-edge limit for this request.
quality: JPEG output quality in the range 1-100.
output_dir: Event working file directory, separate from the shared cache.
output_dir: Event working file directory.
prepared: Per-request mapping reused after the request hook.
quote_image_ref: Optional input for the dedicated quote caption branch.
montage_max_size: Optional montage-specific limit; defaults to ``max_size``.
"""
req.image_urls = normalize_and_dedupe_strings(req.image_urls)
refs = list(req.image_urls)
@@ -53,37 +42,15 @@ async def prepare_request_images(
for ref in dict.fromkeys(refs):
if ref not in prepared:
path = None
if enabled:
image = await prepare_model_image(
ref,
max_size=max_size,
output_dir=output_dir,
quality=quality,
montage_max_size=montage_max_size,
)
if image:
path, is_montage = image
event.track_temporary_local_file(path)
has_montage |= is_montage
else:
try:
async with MediaResolver(
ref, media_type="image"
).as_path() as source:
# Transfer only resolver-owned downloads, never user files.
if not source.path.is_file():
raise FileNotFoundError("Image source is unavailable")
path = str(source.path.resolve())
for owned_path in source.cleanup_paths:
event.track_temporary_local_file(str(owned_path))
source.detach()
except Exception as exc:
if not is_recoverable_image_error(exc):
raise
logger.warning(
"Image localization failed; skipping image (%s).",
type(exc).__name__,
)
image = await prepare_model_image(
ref,
max_size=max_size,
output_dir=output_dir,
)
if image:
path, is_montage = image
event.track_temporary_local_file(path)
has_montage |= is_montage
prepared[ref] = path
if path:
prepared[path] = path
@@ -43,10 +43,7 @@ from astrbot.core.provider.entities import (
)
from astrbot.core.star.star_handler import EventType
from astrbot.core.utils.astrbot_path import get_astrbot_temp_path
from astrbot.core.utils.media_utils import (
IMAGE_COMPRESS_DEFAULT_QUALITY,
normalize_model_image_max_size,
)
from astrbot.core.utils.media_utils import normalize_model_image_max_size
from astrbot.core.utils.metrics import Metric
from astrbot.core.utils.session_lock import session_lock_manager
@@ -62,9 +59,6 @@ from ...follow_up import (
)
from .image_input import prepare_request_images
# Anthropic rejects images above 5 MB; OpenAI and Gemini allow roughly 20 MB.
_CUA_IMAGE_WARN_BYTES = 5 * 1024 * 1024
class InternalAgentSubStage(Stage):
async def initialize(self, ctx: PipelineContext) -> None:
@@ -244,32 +238,10 @@ class InternalAgentSubStage(Stage):
if req is None:
return
settings = self.ctx.astrbot_config["provider_settings"]
enabled = settings.get("image_compress_enabled", True) is not False
options = settings.get("image_compress_options", {})
montage_max_size = normalize_model_image_max_size(
max_size = normalize_model_image_max_size(
options.get("max_size") if isinstance(options, dict) else None
)
max_size = montage_max_size
sandbox_cfg = settings.get("sandbox")
cua_pixel_mode = (
settings.get("computer_use_runtime") == "sandbox"
and isinstance(sandbox_cfg, dict)
and sandbox_cfg.get("booter") == "cua"
)
if cua_pixel_mode:
# CUA pixel tools read coordinates 1:1 on stills, so the
# still-image resize is lifted; compliant images pass through
# byte-exact since lossy re-encoding would shift colors.
# Montages are never used for coordinates and keep the
# configured cap, which bounds the 3x3 canvas. Oversized
# passthrough images warn below.
max_size = 1_000_000
quality = (
options.get("quality") if isinstance(options, dict) else None
)
if isinstance(quality, bool) or not isinstance(quality, int):
quality = IMAGE_COMPRESS_DEFAULT_QUALITY
quality = min(max(quality, 1), 100)
output_dir = Path(get_astrbot_temp_path())
prepared: dict[str, str | None] = {}
supports_image = _provider_supports_modality(provider, "image")
@@ -287,13 +259,10 @@ class InternalAgentSubStage(Stage):
await prepare_request_images(
req,
event,
enabled=enabled,
max_size=max_size,
quality=quality,
output_dir=output_dir,
prepared=prepared,
quote_image_ref=quote_image_ref,
montage_max_size=montage_max_size,
)
await _process_quote_message(
event,
@@ -355,31 +324,10 @@ class InternalAgentSubStage(Stage):
await prepare_request_images(
req,
event,
enabled=enabled,
max_size=max_size,
quality=quality,
output_dir=output_dir,
prepared=prepared,
montage_max_size=montage_max_size,
)
if cua_pixel_mode:
oversized = []
for path in {p for p in prepared.values() if p}:
try:
size = Path(path).stat().st_size
except OSError:
continue
if size > _CUA_IMAGE_WARN_BYTES:
oversized.append(size)
if oversized:
logger.warning(
"CUA session sends %d image(s) larger than %d MB "
"(largest %.1f MB) without resize; this may exceed "
"provider image upload limits.",
len(oversized),
_CUA_IMAGE_WARN_BYTES // 1048576,
max(oversized) / 1048576,
)
# apply reset
if reset_coro:
await reset_coro
+136 -323
View File
@@ -8,7 +8,6 @@ import asyncio
import base64
import binascii
import errno
import hashlib
import io
import math
import mimetypes
@@ -82,9 +81,6 @@ ANIMATED_MONTAGE_GRID = 3
ANIMATED_MONTAGE_FRAME_COUNT = ANIMATED_MONTAGE_GRID * ANIMATED_MONTAGE_GRID
CONVERT_CACHE_DIR_NAME = "media_convert_cache"
"""Cache directory (under the AstrBot temp dir) for converted images and frames."""
AUDIO_FORMAT_MIME_TYPES = {
"aac": "audio/aac",
"amr": "audio/amr",
@@ -936,11 +932,6 @@ async def resolve_image_ref_to_base64_data(
)
def _image_convert_cache_dir() -> Path:
# Reading and encoding must work even when the optional cache is unavailable.
return Path(get_astrbot_temp_path()) / CONVERT_CACHE_DIR_NAME
def is_recoverable_image_error(error: Exception) -> bool:
"""Identify ordinary input, decoder, network and cache failures.
@@ -971,55 +962,8 @@ def is_recoverable_image_error(error: Exception) -> bool:
)
def _image_convert_cache_key(source_bytes: bytes, params: str) -> str:
"""Build a content-addressed key for derived image bytes.
Args:
source_bytes: Original encoded image content.
params: Algorithm version, output category and effective size.
Returns:
A filesystem-safe content and parameter digest.
"""
source_digest = hashlib.sha256(source_bytes).hexdigest()[:32]
params_digest = hashlib.sha256(params.encode()).hexdigest()[:8]
return f"{source_digest}_{params_digest}"
def _image_has_alpha(image: PILImage.Image) -> bool:
return "A" in image.getbands() or "transparency" in image.info
def _inspect_image(image_bytes: bytes) -> int:
"""Inspect decodable image bytes.
Args:
image_bytes: Encoded image bytes.
Returns:
Number of frames in the validated image.
Raises:
Exception: Raised by Pillow when the bytes are not a decodable image.
"""
with PILImage.open(io.BytesIO(image_bytes)) as image:
frame_count = getattr(image, "n_frames", 1)
image.verify()
if frame_count == 1:
# verify() checks structure, not pixels. Reopen before loading; compatible
# still images otherwise bypass every operation that decodes their data.
with PILImage.open(io.BytesIO(image_bytes)) as image:
image.load()
# Animation frames are decoded while building the montage.
return frame_count
_IMAGE_CONVERT_CACHE_VERSION = "v9-icc"
"""Bump when conversion output semantics change (modes, transparency, sizing)
so stale cache entries produced by older code are never served."""
MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES = 1024 * 1024
"""PNG outputs larger than this are flattened onto white and re-encoded as JPEG."""
MODEL_IMAGE_MAX_BYTES = 512 * 1024
"""Model input image files must be strictly smaller than this limit."""
def normalize_model_image_max_size(value: object) -> int:
@@ -1066,195 +1010,104 @@ def normalize_model_image_max_size(value: object) -> int:
def _encode_image_frame_bytes(
image: PILImage.Image,
max_size: int | None = None,
quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY,
) -> bytes:
"""Encode a display-oriented frame as JPEG, or PNG when it carries transparency.
"""Encode an owned frame below the byte limit while preserving transparency.
Args:
image: Opened source frame, which is never mutated.
image: Decoded frame owned by the caller; its pixels and metadata may change.
max_size: Optional longest-edge limit; smaller images are not enlarged.
quality: JPEG output quality in the range 1-100.
Returns:
Encoded single-frame JPEG or PNG bytes. A PNG larger than
MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES is flattened onto white and
re-encoded as JPEG to bound the request payload size. High-bit-depth
samples are min-max normalized to 8-bit before JPEG encoding.
JPEG or transparent PNG bytes strictly below MODEL_IMAGE_MAX_BYTES.
Raises:
ValueError: The image cannot fit the byte limit even at one pixel.
"""
oriented = ImageOps.exif_transpose(image)
prepared = oriented
extras: list[PILImage.Image] = []
ImageOps.exif_transpose(image, in_place=True)
has_alpha = "A" in image.getbands() or "transparency" in image.info
icc_profile = image.info.get("icc_profile")
if image.mode not in {"RGB", "RGBA", "L", "LA", "P", "1", "I", "I;16"}:
# A source profile is invalid after a color-space conversion such as CMYK.
icc_profile = None
prepared = image
try:
has_alpha = _image_has_alpha(oriented)
if has_alpha:
# Promote RGB/L color-key transparency as well as palette alpha.
prepared = oriented.convert("RGBA")
elif oriented.mode not in {"1", "L", "LA", "P", "RGB", "RGBA", "I", "I;16"}:
prepared = oriented.convert("RGB")
if max_size is not None and max(prepared.size) > max_size:
# Pillow forces nearest-neighbor resampling for palette and binary
# images, regardless of the requested filter.
if prepared.mode in {"P", "1"}:
prepared = prepared.convert("RGB" if prepared.mode == "P" else "L")
# Pillow's integer pre-reduction does not support I;16, although
# direct Lanczos resampling does. Keep the original bit depth.
if image.mode != "RGBA":
prepared = image.convert("RGBA")
elif image.mode not in {"RGB", "L", "I", "I;16"}:
prepared = image.convert("L" if image.mode == "1" else "RGB")
if max_size is not None:
prepared.thumbnail(
(max_size, max_size),
PILImage.Resampling.LANCZOS,
reducing_gap=None if prepared.mode == "I;16" else 2.0,
)
# JPEG saving does not auto-embed the source ICC profile like PNG does;
# attach it explicitly, but only when the color space survived intact
# (conversions like CMYK -> RGB invalidate the source profile).
icc_profile = image.info.get("icc_profile")
if image.mode not in {"RGB", "RGBA", "L", "LA", "P", "1", "I", "I;16"}:
icc_profile = None
save_kwargs = {"icc_profile": icc_profile} if icc_profile else {}
if has_alpha:
buffer = io.BytesIO()
prepared.save(buffer, "PNG", **save_kwargs)
data = buffer.getvalue()
if len(data) <= MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES:
return data
# Oversized PNG flattens onto white and re-encodes as JPEG.
rgba = prepared if prepared.mode == "RGBA" else prepared.convert("RGBA")
extras.append(rgba)
flattened = PILImage.new("RGB", rgba.size, (255, 255, 255))
extras.append(flattened)
flattened.paste(rgba, mask=rgba.getchannel("A"))
prepared = flattened
elif prepared.mode in {"I", "I;16"}:
# convert() clips high-bit-depth samples at 255 instead of scaling;
# min-max normalize to 8-bit so the content survives JPEG.
if prepared.mode in {"I", "I;16"}:
# Normalize high-bit-depth pixels instead of clipping them to 255.
low, high = prepared.getextrema()
if high > low:
scaled = prepared.point(lambda v: (v - low) * (255.0 / (high - low)))
extras.append(scaled)
prepared = scaled
prepared = prepared.convert("L")
elif prepared.mode not in {"RGB", "L"}:
# JPEG output supports only RGB and L among the remaining modes.
prepared = prepared.convert("L" if prepared.mode == "1" else "RGB")
buffer = io.BytesIO()
prepared.save(buffer, "JPEG", quality=quality, optimize=True, **save_kwargs)
return buffer.getvalue()
with prepared.point(
lambda v: (v - low) * (255.0 / (high - low))
) as scaled:
prepared = scaled.convert("L")
else:
prepared = prepared.convert("L")
# Do not carry EXIF, text chunks, or other unbounded metadata into previews.
prepared.info.clear()
save_kwargs = {"icc_profile": icc_profile} if icc_profile else {}
while True:
with io.BytesIO() as buffer:
if has_alpha:
prepared.save(buffer, "PNG", **save_kwargs)
else:
prepared.save(
buffer, "JPEG", quality=85, optimize=True, **save_kwargs
)
data = buffer.getvalue()
if len(data) < MODEL_IMAGE_MAX_BYTES:
return data
if save_kwargs:
# Oversized profiles must not defeat the byte limit.
save_kwargs.clear()
continue
if prepared.size == (1, 1):
raise ValueError("Image cannot fit the model input byte limit")
scale = min(0.85, math.sqrt((MODEL_IMAGE_MAX_BYTES - 1) / len(data)) * 0.95)
prepared.thumbnail(
(
max(1, int(prepared.width * scale)),
max(1, int(prepared.height * scale)),
),
PILImage.Resampling.LANCZOS,
)
finally:
for extra in extras:
extra.close()
if prepared is not oriented and prepared not in extras:
if prepared is not image:
prepared.close()
oriented.close()
def _read_valid_cached_image_bytes(output_path: Path) -> bytes | None:
"""Read a cache entry, treating any miss or corruption as absent.
The temp cleaner may delete cache entries concurrently: a missing,
unreadable, empty, or undecodable entry is treated as a cache miss and
the caller rebuilds from the source bytes it already holds.
def _prepare_model_image_sync(source_bytes: bytes, max_size: int) -> tuple[bytes, bool]:
"""Prepare a preview using a single opened source image.
Args:
output_path: Candidate cache file path.
source_bytes: Original encoded image content.
max_size: Longest-edge limit for still images and animation montages.
Returns:
Decodable cached bytes, or ``None`` when the entry is unusable.
"""
try:
data = output_path.read_bytes()
except OSError as exc:
if not is_recoverable_image_error(exc):
raise
return None
if not data:
return None
try:
frame_count = _inspect_image(data)
with PILImage.open(io.BytesIO(data)) as image:
if (
image.format not in {"PNG", "JPEG"}
or frame_count != 1
or image.getexif().get(274, 1) != 1
):
return None
except Exception as exc:
if not is_recoverable_image_error(exc):
raise
return None
return data
def _publish_image_cache_atomic(
output_path: Path,
data: bytes,
) -> None:
"""Publish converted bytes via a unique temp file and atomic replace.
Concurrent readers never observe a partially written cache file. Cache
publication failures (e.g. the cache directory vanished mid-write) are
non-fatal: callers already hold the complete encoded bytes.
Args:
output_path: Final cache entry path.
data: Complete encoded bytes to publish.
"""
try:
output_path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_name = tempfile.mkstemp(dir=output_path.parent, suffix=".tmp")
except OSError as exc:
if not is_recoverable_image_error(exc):
raise
logger.debug("Failed to create image cache temp file: %s", exc)
return
tmp_path = Path(tmp_name)
try:
with os.fdopen(fd, "wb") as temp_file:
temp_file.write(data)
os.replace(tmp_path, output_path)
except OSError as exc:
if not is_recoverable_image_error(exc):
raise
logger.debug("Failed to publish image cache %s: %s", output_path, exc)
finally:
try:
tmp_path.unlink(missing_ok=True)
except OSError as exc:
if not is_recoverable_image_error(exc):
raise
logger.debug("Failed to clean up image cache temp file: %s", exc)
def _convert_image_bytes_sync(
source_bytes: bytes, max_size: int, quality: int
) -> bytes:
"""Normalize a validated still image with an optional derived cache.
Args:
source_bytes: Encoded source bytes already checked by _inspect_image.
max_size: Longest-edge limit in pixels.
quality: JPEG output quality in the range 1-100.
Returns:
Single-frame JPEG or PNG bytes. An oriented JPEG or PNG within the size
limit is reused unchanged; anything else is re-encoded.
Encoded image bytes and whether they represent an animation montage.
"""
with PILImage.open(io.BytesIO(source_bytes)) as image:
if getattr(image, "n_frames", 1) > 1:
return _extract_animation_montage_sync(image, max_size), True
if (
image.format in {"PNG", "JPEG"}
and image.getexif().get(274, 1) == 1
and len(source_bytes) < MODEL_IMAGE_MAX_BYTES
and max(image.size) <= max_size
and image.getexif().get(274, 1) == 1
):
return source_bytes
cache_key = _image_convert_cache_key(
source_bytes,
f"{_IMAGE_CONVERT_CACHE_VERSION}|convert|s={max_size}|q={quality}",
)
output_path = _image_convert_cache_dir() / (cache_key + ".img")
cached = _read_valid_cached_image_bytes(output_path)
if cached is not None:
return cached
encoded = _encode_image_frame_bytes(image, max_size=max_size, quality=quality)
_publish_image_cache_atomic(output_path, encoded)
return encoded
# Validate even byte-identical passthroughs without decoding twice.
image.load()
return source_bytes, False
return _encode_image_frame_bytes(image, max_size), False
def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]:
@@ -1273,91 +1126,70 @@ def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]:
return sorted({round(i * (total_frames - 1) / (count - 1)) for i in range(count)})
def _extract_animation_montage_sync(
source_bytes: bytes,
max_size: int,
quality: int,
) -> tuple[bytes, bool]:
"""Tile evenly spaced frames of an animated image into one grid montage.
Frame sampling includes the first and last frames; grid cells beyond the
available frames stay blank (white). The montage is flattened onto a white
background and its longest edge is capped at ``max_size`` (never upscaled).
def _extract_animation_montage_sync(image: PILImage.Image, max_size: int) -> bytes:
"""Tile evenly spaced animation frames into one white 3x3 preview.
Args:
source_bytes: Encoded animated image bytes.
max_size: Longest edge of the montage in pixels.
image: Opened animation owned by the caller.
max_size: Longest-edge limit of the montage, without upscaling.
Returns:
Tuple of the montage bytes and whether extraction actually ran
(``False`` when a valid cached montage was served).
JPEG bytes below the byte limit, with unused cells left white.
"""
cache_key = _image_convert_cache_key(
source_bytes,
f"{_IMAGE_CONVERT_CACHE_VERSION}|montage|s={max_size}|q={quality}",
total_frames = getattr(image, "n_frames", 1)
# APNG's independent default image is a cover, not an animation frame.
first_frame = 1 if image.info.get("default_image", False) else 0
frame_indices = [
first_frame + index
for index in _even_frame_indices(
total_frames - first_frame, ANIMATED_MONTAGE_FRAME_COUNT
)
]
image.seek(first_frame)
with ImageOps.exif_transpose(image) as oriented:
display_size = oriented.size
# Floor the per-cell scale so the montage never exceeds max_size.
longest_edge = max(display_size) * ANIMATED_MONTAGE_GRID
scale = min(1.0, max(max_size, 1) / longest_edge)
cell_size = (
max(1, int(display_size[0] * scale)),
max(1, int(display_size[1] * scale)),
)
output_path = _image_convert_cache_dir() / (cache_key + ".img")
cached = _read_valid_cached_image_bytes(output_path)
if cached is not None:
return cached, False
with PILImage.open(io.BytesIO(source_bytes)) as image:
total_frames = getattr(image, "n_frames", 1)
# APNG's independent default image is a cover, not an animation frame.
first_frame = 1 if image.info.get("default_image", False) else 0
frame_indices = [
first_frame + index
for index in _even_frame_indices(
total_frames - first_frame, ANIMATED_MONTAGE_FRAME_COUNT
)
]
image.seek(first_frame)
with ImageOps.exif_transpose(image) as oriented:
display_size = oriented.size
# Floor the per-cell scale so the montage never exceeds max_size.
longest_edge = max(display_size) * ANIMATED_MONTAGE_GRID
scale = min(1.0, max(max_size, 1) / longest_edge)
cell_size = (
max(1, int(display_size[0] * scale)),
max(1, int(display_size[1] * scale)),
)
canvas = PILImage.new(
"RGB",
(
cell_size[0] * ANIMATED_MONTAGE_GRID,
cell_size[1] * ANIMATED_MONTAGE_GRID,
),
(255, 255, 255),
)
try:
for out_index, frame_index in enumerate(frame_indices):
image.seek(frame_index)
with (
ImageOps.exif_transpose(image) as oriented,
oriented.convert("RGBA") as frame,
):
resized = frame
try:
if frame.size != cell_size:
resized = frame.resize(
cell_size, PILImage.Resampling.LANCZOS
)
# Alpha shows the white canvas through transparent pixels.
canvas.paste(
resized,
(
(out_index % ANIMATED_MONTAGE_GRID) * cell_size[0],
(out_index // ANIMATED_MONTAGE_GRID) * cell_size[1],
),
resized,
)
finally:
if resized is not frame:
resized.close()
encoded = _encode_image_frame_bytes(canvas, quality=quality)
finally:
canvas.close()
_publish_image_cache_atomic(output_path, encoded)
return encoded, True
canvas = PILImage.new(
"RGB",
(
cell_size[0] * ANIMATED_MONTAGE_GRID,
cell_size[1] * ANIMATED_MONTAGE_GRID,
),
(255, 255, 255),
)
try:
for out_index, frame_index in enumerate(frame_indices):
image.seek(frame_index)
with (
ImageOps.exif_transpose(image) as oriented,
oriented.convert("RGBA") as frame,
):
resized = frame
try:
if frame.size != cell_size:
resized = frame.resize(cell_size, PILImage.Resampling.LANCZOS)
# Alpha shows the white canvas through transparent pixels.
canvas.paste(
resized,
(
(out_index % ANIMATED_MONTAGE_GRID) * cell_size[0],
(out_index // ANIMATED_MONTAGE_GRID) * cell_size[1],
),
resized,
)
finally:
if resized is not frame:
resized.close()
encoded = _encode_image_frame_bytes(canvas)
finally:
canvas.close()
return encoded
async def prepare_model_image(
@@ -1365,44 +1197,25 @@ async def prepare_model_image(
*,
max_size: int,
output_dir: Path,
quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY,
montage_max_size: int | None = None,
) -> tuple[str, bool] | None:
"""Prepare a single local model-ready image for the caller to own until consumption.
Args:
image_ref: Source reference accepted by MediaResolver.
max_size: Validated longest-edge limit for stills.
output_dir: Directory for independent request-owned working files.
quality: JPEG output quality in the range 1-100.
montage_max_size: Optional longest-edge limit for animation montages.
CUA sessions lift the still-image cap to keep pixel coordinates 1:1,
but montages are never used for coordinates, so callers pass the
configured limit here to keep the 3x3 canvas bounded. Defaults to
``max_size``.
max_size: Longest-edge limit for both still images and montages.
output_dir: Directory for event-owned working files.
Returns:
Tuple of an existing JPEG or PNG path and whether it is an animation
frame montage (including cache hits), or None for a recoverable input
or write failure.
The caller owns this file; shared cache entries are never returned.
A preview path and whether it is an animation montage, or None for a
recoverable input or write failure. The caller must delete the preview
after use. The original image is never overwritten.
"""
try:
async with MediaResolver(image_ref, media_type="image").as_path() as source:
image_bytes = await asyncio.to_thread(source.read_bytes)
frame_count = await asyncio.to_thread(_inspect_image, image_bytes)
is_montage = frame_count > 1
if is_montage:
converted_bytes, _ = await asyncio.to_thread(
_extract_animation_montage_sync,
image_bytes,
montage_max_size if montage_max_size is not None else max_size,
quality,
)
else:
converted_bytes = await asyncio.to_thread(
_convert_image_bytes_sync, image_bytes, max_size, quality
)
converted_bytes, is_montage = await asyncio.to_thread(
_prepare_model_image_sync, image_bytes, max_size
)
# Publish the working file synchronously after encoding, so cancellation
# cannot leave an untracked background write alive after this call.
output_dir.mkdir(parents=True, exist_ok=True)
+4
View File
@@ -0,0 +1,4 @@
# Unreleased
- Local Agent input images are always prepared as JPEG/PNG files strictly below 512 KiB. Transparent previews retain PNG alpha; animations become 3×3 montages. Original attachment paths remain available to tools, and event-owned previews are deleted after use without a shared conversion cache.
- Configuration mapping: **Enable image compression** (`provider_settings.image_compress_enabled`) is replaced by always-on preparation; **JPEG quality** (`provider_settings.image_compress_options.quality`) is replaced by automatic size control; **Maximum edge length** (`provider_settings.image_compress_options.max_size`) remains available. User attachments in CUA sessions follow the same limits.
@@ -481,20 +481,12 @@
"description": "User Prompt",
"hint": "You can use {{prompt}} as a placeholder for user input. If no placeholder is provided, it will be added before the user input."
},
"image_compress_enabled": {
"description": "Enable image compression",
"hint": "Enabled by default. Images are resized or converted as needed before being sent to multimodal models; compliant JPEG/PNG images are sent unchanged, and animations become a preview grid."
},
"image_compress_options": {
"description": "Image compression settings",
"hint": "Control the size limit and JPEG output quality for model input images.",
"description": "Model input image settings",
"hint": "Images are always prepared as JPEG/PNG below 512 KiB. Original attachments are preserved.",
"max_size": {
"description": "Maximum edge length",
"hint": "Longest edge of the compressed image in pixels. Images larger than this are resized proportionally."
},
"quality": {
"description": "JPEG quality",
"hint": "JPEG output quality from 1 to 100. Higher values preserve more detail but produce larger files."
"hint": "Longest edge sent to the model, in pixels. Images may be reduced further to stay below 512 KiB; original attachment paths are unchanged."
}
},
"reachability_check": {
@@ -582,20 +582,12 @@
"description": "ユーザープロンプト",
"hint": "{{prompt}} をユーザー入力のプレースホルダーとして使用できます。プレースホルダーを指定しない場合は、ユーザー入力の前に追加されます。"
},
"image_compress_enabled": {
"description": "画像圧縮を有効化",
"hint": "デフォルトで有効。マルチモーダルモデルへの送信前に画像を必要に応じて圧縮・変換します。条件を満たす JPEG/PNG はそのまま送信され、アニメーションはプレビューグリッドになります。"
},
"image_compress_options": {
"description": "画像圧縮設定",
"hint": "モデルへ送信する画像のサイズ上限と JPEG 出力品質を設定します。",
"description": "モデル入力画像の設定",
"hint": "画像は常に 512 KiB 未満の JPEG/PNG に処理されます。元の添付ファイルは保持されます。",
"max_size": {
"description": "最大辺長",
"hint": "圧縮後の画像の長辺をピクセル単位で指定します。このサイズを超える場合は縦横比を維持して縮小します。"
},
"quality": {
"description": "JPEG 品質",
"hint": "JPEG の出力品質を 1~100 の範囲で指定します。値が高いほど画質が向上し、ファイルサイズも大きくなります。"
"hint": "モデルに送信する画像の最長辺(ピクセル)。512 KiB 未満にするため、さらに縮小される場合があります。元の添付ファイルのパスは変わりません。"
}
},
"reachability_check": {
@@ -621,20 +621,12 @@
"description": "Промпт пользователя",
"hint": "Вы можете использовать {{prompt}} как заполнитель для ввода. Если заполнитель не указан, он будет добавлен перед текстом пользователя."
},
"image_compress_enabled": {
"description": "Включить сжатие изображений",
"hint": "Включено по умолчанию. Перед отправкой мультимодальным моделям изображения при необходимости сжимаются или преобразуются; корректные JPEG/PNG отправляются без изменений, анимации собираются в сетку предпросмотра."
},
"image_compress_options": {
"description": "Настройки сжатия изображений",
"hint": "Управляет ограничением размера изображений для модели и качеством JPEG.",
"description": "Настройки изображений для модели",
"hint": "Изображения всегда преобразуются в JPEG/PNG размером менее 512 КиБ. Исходные вложения сохраняются.",
"max_size": {
"description": "Максимальная длина стороны",
"hint": "Максимальная длина стороны сжатого изображения в пикселях. Более крупные изображения пропорционально уменьшаются."
},
"quality": {
"description": "Качество JPEG",
"hint": "Качество JPEG от 1 до 100. Более высокие значения сохраняют больше деталей, но увеличивают размер файла."
"hint": "Максимальная длина стороны изображения для модели в пикселях. При необходимости изображение уменьшается дополнительно до размера менее 512 КиБ. Путь к исходному вложению не меняется."
}
},
"reachability_check": {
@@ -502,20 +502,12 @@
"description": "用户提示词",
"hint": "可使用 {{prompt}} 作为用户输入的占位符。如果不输入占位符则代表添加在用户输入的前面。"
},
"image_compress_enabled": {
"description": "启用图片压缩",
"hint": "默认开启。发送给多模态模型前按需压缩转换图片:合规的 JPEG/PNG 原样发送,动图生成拼图预览。"
},
"image_compress_options": {
"description": "图片压缩配置",
"hint": "用于控制发送给模型的图片尺寸上限和 JPEG 输出质量。",
"description": "模型输入图片设置",
"hint": "图片始终处理为严格小于 512 KiB 的 JPEG/PNG,原始附件保留。",
"max_size": {
"description": "最大边长",
"hint": "压缩后图片的最长边,单位为像素。超过该尺寸时会按比例缩放。"
},
"quality": {
"description": "JPEG 质量",
"hint": "JPEG 输出质量,范围为 1-100。值越高,画质越好,文件也越大。"
"hint": "发送给模型的图片最长边,单位为像素。为保证单张图片小于 512 KiB,必要时会进一步缩小;原始附件路径保持不变。"
}
},
"reachability_check": {
+24 -17
View File
@@ -1,28 +1,35 @@
# Model input images in the local ProcessStage
# Model input images
“Enable image compression” (`provider_settings.image_compress_enabled`) is enabled by default. The local Agent branch of ProcessStage prepares the current input before building the Agent, including ordinary attachments, quoted images and plugin `ProviderRequest` images. It checks images added or replaced by `OnLLMRequestEvent` once more before Runner reset.
The local Agent always prepares current input images before sending them to the model. This covers ordinary attachments, quoted images, and plugin `ProviderRequest` images, including images added or replaced by `OnLLMRequestEvent`.
## Preparation rules
## Limits and formats
- Compliant still images are sent unchanged: already JPEG or PNG, correctly oriented, and within the size limit.
- Other still images are orientation-corrected, resized and re-encoded as needed. Opaque images become JPEG, controlled by `image_compress_options.quality` (default 95), with high-bit-depth samples normalized to 8-bit. Images with transparency become PNG; a PNG larger than 1 MB is flattened onto a white background and re-encoded as JPEG to bound the payload size.
- Animations are detected from their frames, including GIF, animated WebP and APNG. Up to nine evenly sampled frames, including the first and last, become one white 3×3 grid. Unused cells stay white; an APNG independent cover is excluded.
- Stills and montages share `image_compress_options.max_size` (default 1280). Small images are not enlarged.
- Disabling compression keeps generic localization and reading, without resizing, transcoding, sampling or consulting the derived-image cache.
- Each prepared image file is **strictly smaller than 512 KiB** (524,288 bytes). This limit applies to image bytes before Base64 encoding, not the entire request.
- **Maximum edge length**, under **Configuration**, remains configurable through `provider_settings.image_compress_options.max_size` (default 1280 pixels). Images keep their aspect ratio and are never enlarged. They may be reduced further to meet the byte limit.
- Correctly oriented JPEG/PNG images already within both limits retain their original bytes. Other still images are orientation-corrected and encoded as JPEG, or PNG when they have transparency. Transparent images retain their alpha channel even when further reduction is necessary.
- GIF, animated WebP, and APNG inputs become a white 3×3 frame montage. Up to nine evenly spaced frames include the first and last animation frames; unused cells stay blank. An independent APNG cover is excluded.
- Unneeded metadata is removed when re-encoding. A color profile is retained when compatible with the output and the byte limit.
These limits also apply to user attachments in CUA sessions. Screenshot and other tool-result images keep their own handling.
> [!TIP]
> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Compliant images pass through byte-exact (no lossy re-encoding, avoiding JPEG color shifts); format conversion for other formats and animation montages still apply. Images above roughly 5 MB may exceed provider image upload limits and trigger a warning in the logs.
The Agent receives readable local paths. Original image files and event components keep their original content, and attachment text continues to reference the source image. Providers only read/encode references and assemble their protocols.
## Configuration changes
## Errors and lifetime
| Previous setting | Current behavior |
| --- | --- |
| Enable image compression (`image_compress_enabled`) | Removed; current input images are always prepared. |
| JPEG quality (`image_compress_options.quality`) | Removed; output size is controlled automatically. |
| Maximum edge length (`image_compress_options.max_size`) | Retained; applies together with the strict 512 KiB limit. |
An unreadable, corrupt or locally undecodable image is skipped with a short warning; valid text and other images remain. If nothing usable remains, request preparation supplies a placeholder. Cancellation, resource exhaustion and programming errors are not treated as bad images.
Values of the removed settings in existing configuration files no longer affect image preparation.
Derived bytes are cached by source content, effective size, JPEG quality, still/montage category and algorithm version. Missing or corrupt entries are rebuilt. Each request receives independent event-owned working files, so event cleanup does not remove shared cache files. An unwritable cache can be bypassed; inability to create the working file skips that image.
## Original files and preview lifetime
Successfully localized event attachments and sources adopted during request collection remain available after event cleanup, including quoted images. Attachment text keeps these source paths; model working copies and request-only downloads are still cleaned up. Newly materialized sources stay event-owned if collection fails or is cancelled. Retained files under the temporary directory remain subject to `temp_dir_max_size` cleanup and are not permanent storage.
`image_urls` and image content parts use the prepared preview files. The attachment paths shown to the model for file access continue to point to the **original images**, so tools can read the originals. Original files and event image components are not overwritten.
Existing message serialization stores the visual content the model received, including prepared images, before temporary paths expire. Replay and fallback reuse that content; old history is not rescanned or migrated.
Preview files are owned by the current event and deleted when the event finishes, including failure or cancellation. There is no shared image-conversion cache. Repeated references within one request reuse the same preview, including the preparation pass after the request hook.
This feature covers only current inputs through the local ProcessStage. Third-party Agent backends, direct plugin calls to Agent/Provider APIs, tool-result images, read_file and CUA retain their existing behavior. Tool-image handling is a separate follow-up and is not required for this feature.
Successfully localized original attachments, including quoted images, survive event cleanup. Retained originals under the temporary directory remain subject to `temp_dir_max_size` cleanup and are not permanent storage. Sources that have not yet been successfully adopted when request collection fails remain event-owned.
Conversation history captures the prepared image content as Base64 data URIs before preview files expire. Old history is not reprocessed. An unreadable image is skipped; valid text and other images remain. Cancellation, resource exhaustion, and programming errors are not treated as bad images.
Third-party Agent backends, direct calls that bypass the local pipeline, and tool-result images are outside this input preparation flow.
+23 -17
View File
@@ -1,29 +1,35 @@
# 图片处理
使用内置 Agent 执行器处理新消息时,AstrBot 默认会在发送图片给模型前按需处理图片,减少格式不兼容的问题。处理不会覆盖原图。
使用内置 Agent 处理新消息时,AstrBot 始终在发送给模型前处理本次输入的图片,包括普通附件、引用图片,以及插件 `ProviderRequest` 中的图片。请求 Hook 新增或替换的图片也会处理。
## 配置
## 大小限制与格式
在管理面板的 **配置文件** 页面搜索 **启用图片压缩**。该选项默认开启,关闭后不再缩放、转码或生成动图拼图。
- 模型使用的单张图片文件**严格小于 512 KiB**(524,288 字节)。限制针对 Base64 编码前的图片文件,不是整个请求体。
- 在管理面板的**配置文件**页面搜索**最大边长**,对应 `provider_settings.image_compress_options.max_size`,默认 1280 像素。图片保持宽高比,小图不会放大;为满足文件体积限制,必要时会进一步缩小。
- 方向正常、像素尺寸及文件体积均合规的 JPEG/PNG 保留原始字节。其余静图修正方向后编码为 JPEG;含透明度的图片输出 PNG,即使需要进一步缩小也保留透明通道。
- GIF、动画 WebP、APNG 等动图均匀选取最多 9 帧,包含首尾动画帧,生成白底 3×3 拼图。不足 9 帧的格子留白,APNG 的独立封面不参与采样。
- 重新编码时移除不必要的元数据;与输出色彩空间兼容、且不影响体积达标的色彩配置保留。
**最大边长**默认为 1280 像素,静图和动图拼图共用此设置,小图不会放大。**压缩质量**(1-100,默认 95)控制 JPEG 输出的画质与体积。
CUA 会话中的用户输入图片也遵守这些限制;截图及其他工具返回的图片继续使用各自的处理逻辑。
## 配置项变化
> [!TIP]
> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时,为避免像素坐标漂移,输入图片不按最大边长缩放,合规图片原样发送(不重新编码,避免 JPEG 变色);格式转换与动图拼图等其余处理仍然生效。超过约 5 MB 的大图可能超出服务商的图片体积限制,此时日志会给出警告。
## 处理方式
| 原配置项 | 现在的行为 |
| --- | --- |
| 启用图片压缩(`image_compress_enabled`) | 已移除,当前输入图片始终处理。 |
| 压缩质量(`image_compress_options.quality`) | 已移除,自动控制输出体积。 |
| 最大边长(`image_compress_options.max_size`) | 保留,同时遵守严格小于 512 KiB 的文件体积限制。 |
- **合规静图原样发送**:已是 JPEG/PNG、方向正常且未超过最大边长的图片不做任何改动。
- **其余静图按需转换**:方向异常、尺寸超限或格式不通用的图片会修正方向、缩放后重新编码。无透明度的图片输出 JPEG(16 位等高位深图片会归一化为 8 位);含透明度的图片输出 PNG,若 PNG 超过 1 MB 则扁平化为白底 JPEG,控制发送体积。
- **动图**:从 GIF、动画 WebP、APNG 等图片中均匀选取最多 9 帧,包含首尾动画帧,生成白底 3×3 拼图;不足 9 帧的格子留白。单帧 GIF 和静态 WebP 按静图处理。
- **无法读取或解码的图片**:跳过该图片,其他文字和正常图片继续处理。
旧配置文件中遗留的启用开关和 JPEG 质量值不再影响图片处理。
> [!TIP]
> 如需更小的请求体积,可调低“最大边长”或“压缩质量”。
## 原图路径与文件清理
## 文件保留与清理
`image_urls` 及图片内容块使用处理后的预览文件。提供给模型、供工具读取文件的附件说明仍然指向**原始图片路径**,不会替换为预览路径。原始文件和事件中的图片组件均不被覆盖。
预处理成功本地化的附件原图,以及请求收集成功接管的原图(含引用图片),不会在事件结束时立即删除,附件说明继续引用这些原图路径。模型使用的派生文件和仅用于请求的下载文件仍随事件清理;请求收集失败或取消时,新生成且尚未接管的原图也会清理。临时目录中的保留文件仍受 `temp_dir_max_size` 容量清理约束,不属于永久存储。
模型预览文件归当前事件所有,事件完成、失败或取消后清理,不再保留跨请求图片转换缓存。同一请求中重复出现的图片引用复用同一份预览,请求 Hook 前后的准备过程也会去重。
> [!NOTE]
> 本功能仅适用于经过本地消息流水线的本次输入,包括引用图片。工具回图、第三方 Agent 后端及插件绕过流水线直接调用模型不在处理范围内。会话历史保存模型实际看到的图片内容,不会重新转换旧图片。
成功本地化的附件原图,包括引用图片,不随事件结束删除。临时目录中的原图仍受 `temp_dir_max_size` 容量清理约束,不属于永久存储。请求收集失败时,尚未成功接管的源文件仍按事件临时文件清理。
对话历史会在预览文件删除前保存模型使用的图片内容,采用 Base64 data URI;旧历史不会重新处理。无法读取的图片会跳过,不影响其他文字和正常图片;取消、资源耗尽及程序错误不会被当作坏图忽略。
第三方 Agent 后端、绕过本地流水线的直接调用,以及工具返回的图片,不属于本次输入图片处理范围。
+72 -69
View File
@@ -2,9 +2,9 @@
import asyncio
import errno
import random
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from PIL import Image
@@ -14,8 +14,8 @@ from astrbot.core.utils.io import DownloadFileHTTPError
@pytest.fixture(autouse=True)
def isolated_cache(tmp_path, monkeypatch):
monkeypatch.setattr(media, "get_astrbot_temp_path", lambda: str(tmp_path / "cache"))
def isolated_temp(tmp_path, monkeypatch):
monkeypatch.setattr(media, "get_astrbot_temp_path", lambda: str(tmp_path / "temp"))
async def _prepare_image_path(*args, **kwargs):
@@ -69,16 +69,21 @@ async def test_rotated_jpeg_is_normalized_to_upright_jpeg(tmp_path):
@pytest.mark.asyncio
async def test_oversized_alpha_png_flattens_to_jpeg(tmp_path, monkeypatch):
monkeypatch.setattr(media, "MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES", 1)
source = tmp_path / "alpha.png"
Image.new("RGBA", (2000, 100), (255, 0, 0, 128)).save(source)
@pytest.mark.parametrize("fmt", ["PNG", "WEBP"])
async def test_transparent_images_shrink_without_losing_alpha(tmp_path, fmt):
source = tmp_path / f"alpha.{fmt.lower()}"
pixels = random.Random(9703).randbytes(1280 * 960 * 3)
image = Image.frombytes("RGB", (1280, 960), pixels)
image.putalpha(128)
image.save(source, fmt, lossless=True)
original = source.read_bytes()
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
assert path and path.endswith(".jpg")
with Image.open(path) as image:
assert image.format == "JPEG" and image.mode == "RGB"
r, g, b = image.getpixel((0, 0))
assert r > 200 and 90 < g < 170 and 90 < b < 170
assert path and Path(path).stat().st_size < 512 * 1024
with Image.open(path) as result:
assert result.format == "PNG" and result.mode == "RGBA"
assert max(result.size) < 1280
assert result.getchannel("A").getextrema() == (128, 128)
assert source.read_bytes() == original
@pytest.mark.asyncio
@@ -124,19 +129,49 @@ async def test_cmyk_icc_profile_is_not_attached_to_rgb_jpeg(tmp_path):
@pytest.mark.asyncio
async def test_jpeg_quality_is_honored_and_cached_separately(tmp_path):
source = tmp_path / "photo.bmp"
Image.new("RGB", (200, 100), "red").save(source)
low = await _prepare_image_path(
str(source), max_size=1280, output_dir=tmp_path, quality=10
)
high = await _prepare_image_path(
str(source), max_size=1280, output_dir=tmp_path, quality=95
)
assert low and high and low != high
low_bytes, high_bytes = Path(low).read_bytes(), Path(high).read_bytes()
assert low_bytes.startswith(b"\xff\xd8") and high_bytes.startswith(b"\xff\xd8")
assert len(low_bytes) < len(high_bytes)
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "WEBP", "BMP"])
async def test_opaque_images_fit_the_byte_limit(tmp_path, fmt):
source = tmp_path / f"noise.{fmt.lower()}"
pixels = random.Random(9703).randbytes(1280 * 960 * 3)
Image.frombytes("RGB", (1280, 960), pixels).save(source, fmt, quality=100)
original = source.read_bytes()
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
assert path and Path(path).stat().st_size < 512 * 1024
with Image.open(path) as result:
assert result.format == "JPEG" and max(result.size) <= 1280
assert source.read_bytes() == original
@pytest.mark.asyncio
@pytest.mark.parametrize("fmt", ["JPEG", "PNG"])
@pytest.mark.parametrize("size", [512 * 1024 - 1, 512 * 1024, 512 * 1024 + 1])
async def test_byte_limit_is_strict_even_for_dimension_compliant_images(
tmp_path, fmt, size
):
source = tmp_path / f"small.{fmt.lower()}"
Image.new("RGB", (32, 16), "red").save(source, fmt)
original = source.read_bytes().ljust(size, b"\0")
source.write_bytes(original)
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
assert path and Path(path).stat().st_size < 512 * 1024
assert (Path(path).read_bytes() == original) == (size < 512 * 1024)
assert source.read_bytes() == original
@pytest.mark.asyncio
@pytest.mark.parametrize("fmt", ["JPEG", "PNG"])
async def test_oversized_metadata_cannot_bypass_byte_limit(tmp_path, fmt):
source = tmp_path / f"metadata.{fmt.lower()}"
image = Image.new("RGBA" if fmt == "PNG" else "RGB", (32, 16), "red")
image.save(source, fmt, icc_profile=random.Random(512).randbytes(600 * 1024))
original = source.read_bytes()
assert len(original) >= 512 * 1024
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
assert path and Path(path).stat().st_size < 512 * 1024
with Image.open(path) as result:
assert result.size == (32, 16)
assert not result.info.get("icc_profile")
assert source.read_bytes() == original
@pytest.mark.asyncio
@@ -273,40 +308,22 @@ def test_size_normalization(value, expected):
@pytest.mark.asyncio
@pytest.mark.parametrize("bad_cache", [b"", b"bad", None, "animated"])
async def test_bad_cache_rebuilt_and_work_files_independent(
tmp_path, monkeypatch, bad_cache
):
async def test_working_files_are_independent_without_shared_cache(tmp_path):
source = tmp_path / "source.bmp"
Image.new("RGB", (32, 16), "red").save(source)
first = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
cache = next((tmp_path / "cache" / media.CONVERT_CACHE_DIR_NAME).glob("*.img"))
if bad_cache is None:
cache.unlink()
elif bad_cache == "animated":
Image.new("RGB", (16, 8), "blue").save(
cache,
"GIF",
save_all=True,
append_images=[Image.new("RGB", (16, 8), "red")],
)
else:
cache.write_bytes(bad_cache)
second = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
output = tmp_path / "output"
first = await _prepare_image_path(str(source), max_size=16, output_dir=output)
second = await _prepare_image_path(str(source), max_size=16, output_dir=output)
assert first != second and Path(first).read_bytes() == Path(second).read_bytes()
assert set(output.iterdir()) == {Path(first), Path(second)}
Path(first).unlink()
assert cache.is_file() and Path(second).is_file()
encode = Mock(side_effect=AssertionError("valid cache must skip encoding"))
monkeypatch.setattr(media, "_encode_image_frame_bytes", encode)
third = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
assert Path(third).read_bytes() == Path(second).read_bytes()
assert Path(second).is_file() and source.is_file()
@pytest.mark.asyncio
async def test_unwritable_cache_bypassed_but_work_file_failure_skips(tmp_path):
async def test_work_file_failure_skips_image(tmp_path):
source = tmp_path / "source.bmp"
Image.new("RGB", (32, 16), "red").save(source)
(tmp_path / "cache").write_text("blocks directory creation")
path = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
assert Path(path).is_file()
assert (
@@ -351,20 +368,6 @@ async def test_fatal_image_failures_propagate(tmp_path, monkeypatch, error):
await _prepare_image_path("ref", max_size=1280, output_dir=tmp_path)
def test_cache_publication_is_atomic_and_closes_file(tmp_path, monkeypatch):
output = tmp_path / "cache.png"
original_replace = media.os.replace
def replace(source, target):
assert Path(source).read_bytes() == b"complete"
assert not output.exists()
original_replace(source, target)
monkeypatch.setattr(media.os, "replace", replace)
media._publish_image_cache_atomic(output, b"complete")
assert output.read_bytes() == b"complete" and not list(tmp_path.glob("*.tmp"))
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [403, 404, 503])
async def test_real_download_status_failure_is_skipped_without_secret_logs(
@@ -387,8 +390,8 @@ async def test_real_download_status_failure_is_skipped_without_secret_logs(
@pytest.mark.asyncio
@pytest.mark.parametrize("count", [2, 12])
async def test_animation_reports_montage_including_cache_hits(tmp_path, count):
"""Animated inputs report a montage even when its bytes are cached."""
async def test_repeated_animation_preparation_reports_montage(tmp_path, count):
"""Repeated animated inputs produce independent montage previews."""
frames = [Image.new("RGB", (12, 8), (index * 20, 0, 0)) for index in range(count)]
source = tmp_path / "animation.gif"
frames[0].save(
@@ -401,11 +404,11 @@ async def test_animation_reports_montage_including_cache_hits(tmp_path, count):
assert prepared and prepared[1] is True
assert Path(prepared[0]).is_file()
cached = await media.prepare_model_image(
repeated = await media.prepare_model_image(
str(source), max_size=1280, output_dir=tmp_path
)
assert cached and cached[1] is True
assert Path(cached[0]).read_bytes() == Path(prepared[0]).read_bytes()
assert repeated and repeated[1] is True
assert Path(repeated[0]).read_bytes() == Path(prepared[0]).read_bytes()
@pytest.mark.asyncio
+123 -50
View File
@@ -5,6 +5,7 @@ import base64
import copy
import inspect
import os
import random
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
@@ -195,7 +196,7 @@ async def process_event(harness, event, *, preprocess_first=False, stage=None):
@pytest.mark.asyncio
@pytest.mark.parametrize("enabled", [None, True, False])
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "BMP", "WEBP", "GIF"])
async def test_profile_toggle_and_preprocess_to_first_model(
async def test_legacy_toggle_does_not_disable_preparation(
harness, tmp_path, monkeypatch, enabled, fmt
):
source = source_image(tmp_path, fmt)
@@ -205,23 +206,13 @@ async def test_profile_toggle_and_preprocess_to_first_model(
harness.config["provider_settings"].pop("image_compress_enabled", None)
else:
harness.config["provider_settings"]["image_compress_enabled"] = enabled
if enabled is False:
monkeypatch.setattr(
media,
"_read_valid_cached_image_bytes",
lambda *args: pytest.fail("disabled must not query derived cache"),
)
await process_event(harness, event, preprocess_first=True)
assert len(harness.captured) == 1
req = harness.captured[0].req
with PILImage.open(req.image_urls[0]) as image:
expected = (
fmt if enabled is False else (fmt if fmt in {"JPEG", "PNG"} else "JPEG")
)
expected = fmt if fmt in {"JPEG", "PNG"} else "JPEG"
assert image.format == expected
assert image.size == (
(90, 45) if enabled is not False and fmt == "GIF" else (60, 30)
)
assert image.size == ((90, 45) if fmt == "GIF" else (60, 30))
if fmt == "PNG":
assert image.getpixel((0, 0))[3] == 128
assert source.read_bytes() == original
@@ -246,10 +237,10 @@ async def test_profile_toggle_and_preprocess_to_first_model(
("local", "cua", "PNG"),
],
)
async def test_cua_runtime_keeps_input_image_geometry(
async def test_all_runtimes_apply_configured_image_limit(
harness, tmp_path, runtime, booter, fmt
):
"""CUA pixel tools read coordinates 1:1, so only the resize is lifted."""
"""User attachments obey the image limit for every computer runtime."""
path = tmp_path / f"big.{fmt.lower()}"
PILImage.new("RGB", (200, 100), "red").save(path, fmt)
original = path.read_bytes()
@@ -259,16 +250,10 @@ async def test_cua_runtime_keeps_input_image_geometry(
await process_event(harness, event, preprocess_first=True)
assert len(harness.captured) == 1
req = harness.captured[0].req
gated = runtime == "sandbox" and booter == "cua"
with PILImage.open(req.image_urls[0]) as image:
assert image.size == ((200, 100) if gated else (90, 45))
if gated:
# A compliant source is reused byte-exact; other formats are still
# re-encoded (to JPEG) without any resize.
expected = fmt if fmt in {"JPEG", "PNG"} else "JPEG"
assert image.format == expected
if gated and fmt == "PNG":
assert Path(req.image_urls[0]).read_bytes() == original
assert image.size == (90, 45)
assert image.format == "JPEG"
assert path.read_bytes() == original
@pytest.mark.asyncio
@@ -280,10 +265,10 @@ async def test_cua_runtime_keeps_input_image_geometry(
("sandbox", "shipyard_neo", True),
],
)
async def test_cua_oversize_image_warns(
async def test_cua_inputs_obey_byte_limit(
harness, tmp_path, monkeypatch, runtime, booter, big
):
"""Byte-exact CUA passthrough warns when an image may exceed upload limits."""
"""Large user inputs are bounded even in CUA sessions."""
dims = (1500, 1500) if big else (60, 30)
path = tmp_path / "shot.png"
if big:
@@ -302,11 +287,11 @@ async def test_cua_oversize_image_warns(
await process_event(harness, event, preprocess_first=True)
assert len(harness.captured) == 1
req = harness.captured[0].req
gated = runtime == "sandbox" and booter == "cua"
if gated:
assert Path(req.image_urls[0]).read_bytes() == original
warned = any("upload limits" in str(call) for call in warning.call_args_list)
assert warned == (big and gated)
assert Path(req.image_urls[0]).stat().st_size < 512 * 1024
with PILImage.open(req.image_urls[0]) as image:
assert max(image.size) <= 90
assert path.read_bytes() == original
assert not any("upload limits" in str(call) for call in warning.call_args_list)
@pytest.mark.asyncio
@@ -336,12 +321,12 @@ async def test_animation_montage_notice_reaches_model(harness, tmp_path):
still = tmp_path / "still.png"
PILImage.new("RGB", (60, 30), "red").save(still)
# The second animated request reuses the montage cache.
# Legacy settings cannot disable animation preparation.
for source, enabled, expected in (
(animated, True, True),
(animated, True, True),
(still, True, False),
(animated, False, False),
(animated, False, True),
):
harness.config["provider_settings"]["image_compress_enabled"] = enabled
await process_event(harness, make_event([Image(file=str(source))]))
@@ -369,7 +354,7 @@ async def test_profile_reload_and_concurrent_requests(harness, tmp_path):
harness, make_event([Image(file=str(source))], session="first")
)
other_config = copy.deepcopy(harness.config)
other_config["provider_settings"]["image_compress_enabled"] = False
other_config["provider_settings"]["image_compress_options"]["max_size"] = 60
other_ctx = SimpleNamespace(
astrbot_config=other_config, plugin_manager=harness.ctx.plugin_manager
)
@@ -383,7 +368,7 @@ async def test_profile_reload_and_concurrent_requests(harness, tmp_path):
),
process_event(
harness,
make_event([Image(file=str(source))], session="off"),
make_event([Image(file=str(source))], session="other"),
stage=other_stage,
),
)
@@ -392,7 +377,8 @@ async def test_profile_reload_and_concurrent_requests(harness, tmp_path):
}
with PILImage.open(results["small"].image_urls[0]) as image:
assert max(image.size) == 30
assert Path(results["off"].image_urls[0]).read_bytes() == source.read_bytes()
with PILImage.open(results["other"].image_urls[0]) as image:
assert max(image.size) == 60
assert harness.provider.provider_config == before
@@ -479,9 +465,8 @@ async def test_plugin_request_extra_metadata_hook_and_history(
assert all(p["image_url"].get("id") != "dict-id" for p in images)
paths = list(event._temporary_local_files)
event.cleanup_temporary_local_files()
cache_files = list((harness.work / media.CONVERT_CACHE_DIR_NAME).glob("*.img"))
assert all(not Path(path).exists() for path in paths)
assert cache_files and all(path.exists() for path in cache_files)
assert not list(harness.work.rglob("model_image_*"))
assert all(
base64.b64decode(p["image_url"]["url"].split(",", 1)[1]).startswith(b"\xff\xd8")
for p in images
@@ -495,6 +480,98 @@ async def test_plugin_request_extra_metadata_hook_and_history(
assert all(part["source"]["media_type"] == "image/jpeg" for part in visual)
@pytest.mark.asyncio
@pytest.mark.parametrize("entry", ["attachment", "quote", "plugin", "hook"])
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "GIF"])
async def test_provider_and_history_keep_bounded_previews_and_original_paths(
harness, tmp_path, monkeypatch, entry, fmt
):
source = tmp_path / f"original.{fmt.lower()}"
pixels = random.Random(9703).randbytes(1280 * 960 * 3)
image = PILImage.frombytes("RGB", (1280, 960), pixels)
if fmt == "PNG":
image.putalpha(128)
if fmt == "GIF":
image.save(
source,
fmt,
save_all=True,
append_images=[image.transpose(PILImage.Transpose.FLIP_LEFT_RIGHT)],
)
else:
image.save(source, fmt, quality=100)
original = source.read_bytes()
assert len(original) >= 512 * 1024
harness.config["provider_settings"]["image_compress_options"] = {
"max_size": 1280,
"quality": 100,
}
harness.config["provider_settings"]["image_compress_enabled"] = False
part = Image(file=str(source))
event = make_event(
[Reply(id="quoted", chain=[part])] if entry == "quote" else [part]
)
if entry in {"plugin", "hook"}:
event = make_event()
original_request = ProviderRequest(
prompt="describe",
image_urls=[str(source)],
conversation=harness.context.conversation_manager.get_conversation.return_value,
extra_user_content_parts=[
TextPart(text=f"[Image Attachment: path {source}]")
],
)
if entry == "plugin":
event.set_extra("provider_request", original_request)
else:
async def hook(event, kind, *args):
if kind == EventType.OnLLMRequestEvent:
request = args[0]
request.image_urls = original_request.image_urls
request.extra_user_content_parts = (
original_request.extra_user_content_parts
)
return False
monkeypatch.setattr(internal, "call_event_hook", hook)
await process_event(harness, event, preprocess_first=True)
req = harness.captured[0].req
preview = Path(req.image_urls[0])
assert preview != source and preview.stat().st_size < 512 * 1024
texts = [
part.text for part in req.extra_user_content_parts if isinstance(part, TextPart)
]
assert any(str(source) in text for text in texts)
assert all(str(preview) not in text for text in texts)
provider_messages = harness.provider.text_chat.await_args.kwargs["contexts"]
history = (
harness.context.conversation_manager.update_conversation.await_args.kwargs[
"history"
]
)
for messages in (provider_messages, history):
images = [
part
for message in messages
if isinstance(message.get("content"), list)
for part in message["content"]
if part["type"] == "image_url"
]
assert images
for part in images:
data_uri = part["image_url"]["url"]
data = base64.b64decode(data_uri.split(",", 1)[1])
assert len(data) < 512 * 1024
assert data == preview.read_bytes()
assert data_uri.startswith(
"data:image/png;" if fmt == "PNG" else "data:image/jpeg;"
)
event.cleanup_temporary_local_files()
assert not preview.exists()
assert source.read_bytes() == original
@pytest.mark.asyncio
@pytest.mark.parametrize("plugin", [False, True])
async def test_caption_first_call_uses_prepared_quote(harness, tmp_path, plugin):
@@ -660,15 +737,11 @@ async def test_localized_reference_lifetime_and_ownership(
path = Path(harness.captured[0].req.image_urls[0])
assert path.is_file()
assert str(source) not in event._temporary_local_files
assert (str(path) in event._temporary_local_files) == (
enabled or reference != "file"
)
assert str(path) in event._temporary_local_files
with PILImage.open(path) as image:
assert image.format == ("JPEG" if enabled else "GIF")
if not enabled:
assert path.read_bytes() == source.read_bytes()
assert image.format == "JPEG"
event.cleanup_temporary_local_files()
assert path.exists() == (reference == "file" and not enabled)
assert not path.exists()
assert source.is_file()
@@ -819,18 +892,18 @@ async def test_attachment_sources_survive_event_cleanup(
visual_path = Path(req.image_urls[0])
assert attachment_path.read_bytes() == original
assert str(attachment_path) not in event._temporary_local_files
assert (visual_path != attachment_path) == enabled
assert visual_path != attachment_path
with PILImage.open(visual_path) as visual_image:
assert visual_image.format == ("JPEG" if enabled else fmt)
assert max(visual_image.size) == (12 if enabled else 60)
assert visual_image.format == "JPEG"
assert max(visual_image.size) == 12
owned = [Path(path) for path in event._temporary_local_files]
assert len(owned) == 1 + int(enabled)
assert len(owned) == 2
assert unrelated in owned
event.cleanup_temporary_local_files()
assert attachment_path.read_bytes() == original
assert all(not path.exists() for path in owned)
assert visual_path.exists() == (not enabled)
assert not visual_path.exists()
@pytest.mark.asyncio