mirror of
https://github.com/AstrBotDevs/AstrBot.git
synced 2026-10-16 15:06:04 +08:00
fix: bound model input images below 512 KiB
This commit is contained in:
@@ -198,10 +198,8 @@ DEFAULT_CONFIG = {
|
||||
"cua_local": CUA_DEFAULT_CONFIG["local"],
|
||||
"cua_api_key": CUA_DEFAULT_CONFIG["api_key"],
|
||||
},
|
||||
"image_compress_enabled": True,
|
||||
"image_compress_options": {
|
||||
"max_size": 1280,
|
||||
"quality": 95,
|
||||
},
|
||||
},
|
||||
"agent_runner": {
|
||||
@@ -3025,9 +3023,6 @@ CONFIG_METADATA_2 = {
|
||||
"provider_settings": {
|
||||
"type": "object",
|
||||
"items": {
|
||||
"image_compress_enabled": {
|
||||
"type": "bool",
|
||||
},
|
||||
"enable": {
|
||||
"type": "bool",
|
||||
},
|
||||
@@ -4209,29 +4204,12 @@ CONFIG_METADATA_3 = {
|
||||
"type": "string",
|
||||
"hint": "如果唤醒前缀为 /, 额外聊天唤醒前缀为 chat,则需要 /chat 才会触发 LLM 请求",
|
||||
},
|
||||
"provider_settings.image_compress_enabled": {
|
||||
"description": "启用图片压缩",
|
||||
"type": "bool",
|
||||
"hint": "默认开启。发送给多模态模型前按需压缩转换图片:合规的 JPEG/PNG 原样发送,动图生成拼图预览。",
|
||||
},
|
||||
"provider_settings.image_compress_options.max_size": {
|
||||
"description": "最大边长",
|
||||
"type": "int",
|
||||
"hint": "压缩后图片的最长边,单位为像素,超出则按比例缩放。CUA 沙箱下输入图片不缩放,大图可能超出服务商上传限制。",
|
||||
"condition": {
|
||||
"provider_settings.image_compress_enabled": True,
|
||||
},
|
||||
"hint": "发送给模型的图片最长边,单位为像素。图片始终自动处理,单张文件严格小于 512 KiB;必要时会进一步缩小。原始附件及其路径保持不变。",
|
||||
"slider": {"min": 256, "max": 4096, "step": 64},
|
||||
},
|
||||
"provider_settings.image_compress_options.quality": {
|
||||
"description": "压缩质量",
|
||||
"type": "int",
|
||||
"hint": "JPEG 输出质量,范围为 1-100。值越高,画质越好,文件也越大。",
|
||||
"condition": {
|
||||
"provider_settings.image_compress_enabled": True,
|
||||
},
|
||||
"slider": {"min": 1, "max": 100, "step": 1},
|
||||
},
|
||||
"provider_settings.prompt_prefix": {
|
||||
"description": "用户提示词",
|
||||
"type": "string",
|
||||
|
||||
@@ -2,15 +2,10 @@
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from astrbot.core import logger
|
||||
from astrbot.core.agent.message import ImageURLPart, TextPart
|
||||
from astrbot.core.platform.astr_message_event import AstrMessageEvent
|
||||
from astrbot.core.provider.entities import ProviderRequest
|
||||
from astrbot.core.utils.media_utils import (
|
||||
MediaResolver,
|
||||
is_recoverable_image_error,
|
||||
prepare_model_image,
|
||||
)
|
||||
from astrbot.core.utils.media_utils import prepare_model_image
|
||||
from astrbot.core.utils.string_utils import normalize_and_dedupe_strings
|
||||
|
||||
|
||||
@@ -18,26 +13,20 @@ async def prepare_request_images(
|
||||
req: ProviderRequest,
|
||||
event: AstrMessageEvent,
|
||||
*,
|
||||
enabled: bool,
|
||||
max_size: int,
|
||||
quality: int,
|
||||
output_dir: Path,
|
||||
prepared: dict[str, str | None],
|
||||
quote_image_ref: str | None = None,
|
||||
montage_max_size: int | None = None,
|
||||
) -> None:
|
||||
"""Replace current images on a working request and track their owned files.
|
||||
|
||||
Args:
|
||||
req: Working request; shared lists and image blocks are copied on write.
|
||||
event: Owner of downloaded source files and prepared working files.
|
||||
enabled: Whether the current pipeline profile enables model image preparation.
|
||||
max_size: Normalized longest-edge limit for this request.
|
||||
quality: JPEG output quality in the range 1-100.
|
||||
output_dir: Event working file directory, separate from the shared cache.
|
||||
output_dir: Event working file directory.
|
||||
prepared: Per-request mapping reused after the request hook.
|
||||
quote_image_ref: Optional input for the dedicated quote caption branch.
|
||||
montage_max_size: Optional montage-specific limit; defaults to ``max_size``.
|
||||
"""
|
||||
req.image_urls = normalize_and_dedupe_strings(req.image_urls)
|
||||
refs = list(req.image_urls)
|
||||
@@ -53,37 +42,15 @@ async def prepare_request_images(
|
||||
for ref in dict.fromkeys(refs):
|
||||
if ref not in prepared:
|
||||
path = None
|
||||
if enabled:
|
||||
image = await prepare_model_image(
|
||||
ref,
|
||||
max_size=max_size,
|
||||
output_dir=output_dir,
|
||||
quality=quality,
|
||||
montage_max_size=montage_max_size,
|
||||
)
|
||||
if image:
|
||||
path, is_montage = image
|
||||
event.track_temporary_local_file(path)
|
||||
has_montage |= is_montage
|
||||
else:
|
||||
try:
|
||||
async with MediaResolver(
|
||||
ref, media_type="image"
|
||||
).as_path() as source:
|
||||
# Transfer only resolver-owned downloads, never user files.
|
||||
if not source.path.is_file():
|
||||
raise FileNotFoundError("Image source is unavailable")
|
||||
path = str(source.path.resolve())
|
||||
for owned_path in source.cleanup_paths:
|
||||
event.track_temporary_local_file(str(owned_path))
|
||||
source.detach()
|
||||
except Exception as exc:
|
||||
if not is_recoverable_image_error(exc):
|
||||
raise
|
||||
logger.warning(
|
||||
"Image localization failed; skipping image (%s).",
|
||||
type(exc).__name__,
|
||||
)
|
||||
image = await prepare_model_image(
|
||||
ref,
|
||||
max_size=max_size,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
if image:
|
||||
path, is_montage = image
|
||||
event.track_temporary_local_file(path)
|
||||
has_montage |= is_montage
|
||||
prepared[ref] = path
|
||||
if path:
|
||||
prepared[path] = path
|
||||
|
||||
@@ -43,10 +43,7 @@ from astrbot.core.provider.entities import (
|
||||
)
|
||||
from astrbot.core.star.star_handler import EventType
|
||||
from astrbot.core.utils.astrbot_path import get_astrbot_temp_path
|
||||
from astrbot.core.utils.media_utils import (
|
||||
IMAGE_COMPRESS_DEFAULT_QUALITY,
|
||||
normalize_model_image_max_size,
|
||||
)
|
||||
from astrbot.core.utils.media_utils import normalize_model_image_max_size
|
||||
from astrbot.core.utils.metrics import Metric
|
||||
from astrbot.core.utils.session_lock import session_lock_manager
|
||||
|
||||
@@ -62,9 +59,6 @@ from ...follow_up import (
|
||||
)
|
||||
from .image_input import prepare_request_images
|
||||
|
||||
# Anthropic rejects images above 5 MB; OpenAI and Gemini allow roughly 20 MB.
|
||||
_CUA_IMAGE_WARN_BYTES = 5 * 1024 * 1024
|
||||
|
||||
|
||||
class InternalAgentSubStage(Stage):
|
||||
async def initialize(self, ctx: PipelineContext) -> None:
|
||||
@@ -244,32 +238,10 @@ class InternalAgentSubStage(Stage):
|
||||
if req is None:
|
||||
return
|
||||
settings = self.ctx.astrbot_config["provider_settings"]
|
||||
enabled = settings.get("image_compress_enabled", True) is not False
|
||||
options = settings.get("image_compress_options", {})
|
||||
montage_max_size = normalize_model_image_max_size(
|
||||
max_size = normalize_model_image_max_size(
|
||||
options.get("max_size") if isinstance(options, dict) else None
|
||||
)
|
||||
max_size = montage_max_size
|
||||
sandbox_cfg = settings.get("sandbox")
|
||||
cua_pixel_mode = (
|
||||
settings.get("computer_use_runtime") == "sandbox"
|
||||
and isinstance(sandbox_cfg, dict)
|
||||
and sandbox_cfg.get("booter") == "cua"
|
||||
)
|
||||
if cua_pixel_mode:
|
||||
# CUA pixel tools read coordinates 1:1 on stills, so the
|
||||
# still-image resize is lifted; compliant images pass through
|
||||
# byte-exact since lossy re-encoding would shift colors.
|
||||
# Montages are never used for coordinates and keep the
|
||||
# configured cap, which bounds the 3x3 canvas. Oversized
|
||||
# passthrough images warn below.
|
||||
max_size = 1_000_000
|
||||
quality = (
|
||||
options.get("quality") if isinstance(options, dict) else None
|
||||
)
|
||||
if isinstance(quality, bool) or not isinstance(quality, int):
|
||||
quality = IMAGE_COMPRESS_DEFAULT_QUALITY
|
||||
quality = min(max(quality, 1), 100)
|
||||
output_dir = Path(get_astrbot_temp_path())
|
||||
prepared: dict[str, str | None] = {}
|
||||
supports_image = _provider_supports_modality(provider, "image")
|
||||
@@ -287,13 +259,10 @@ class InternalAgentSubStage(Stage):
|
||||
await prepare_request_images(
|
||||
req,
|
||||
event,
|
||||
enabled=enabled,
|
||||
max_size=max_size,
|
||||
quality=quality,
|
||||
output_dir=output_dir,
|
||||
prepared=prepared,
|
||||
quote_image_ref=quote_image_ref,
|
||||
montage_max_size=montage_max_size,
|
||||
)
|
||||
await _process_quote_message(
|
||||
event,
|
||||
@@ -355,31 +324,10 @@ class InternalAgentSubStage(Stage):
|
||||
await prepare_request_images(
|
||||
req,
|
||||
event,
|
||||
enabled=enabled,
|
||||
max_size=max_size,
|
||||
quality=quality,
|
||||
output_dir=output_dir,
|
||||
prepared=prepared,
|
||||
montage_max_size=montage_max_size,
|
||||
)
|
||||
if cua_pixel_mode:
|
||||
oversized = []
|
||||
for path in {p for p in prepared.values() if p}:
|
||||
try:
|
||||
size = Path(path).stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
if size > _CUA_IMAGE_WARN_BYTES:
|
||||
oversized.append(size)
|
||||
if oversized:
|
||||
logger.warning(
|
||||
"CUA session sends %d image(s) larger than %d MB "
|
||||
"(largest %.1f MB) without resize; this may exceed "
|
||||
"provider image upload limits.",
|
||||
len(oversized),
|
||||
_CUA_IMAGE_WARN_BYTES // 1048576,
|
||||
max(oversized) / 1048576,
|
||||
)
|
||||
# apply reset
|
||||
if reset_coro:
|
||||
await reset_coro
|
||||
|
||||
+136
-323
@@ -8,7 +8,6 @@ import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import errno
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
import mimetypes
|
||||
@@ -82,9 +81,6 @@ ANIMATED_MONTAGE_GRID = 3
|
||||
|
||||
ANIMATED_MONTAGE_FRAME_COUNT = ANIMATED_MONTAGE_GRID * ANIMATED_MONTAGE_GRID
|
||||
|
||||
CONVERT_CACHE_DIR_NAME = "media_convert_cache"
|
||||
"""Cache directory (under the AstrBot temp dir) for converted images and frames."""
|
||||
|
||||
AUDIO_FORMAT_MIME_TYPES = {
|
||||
"aac": "audio/aac",
|
||||
"amr": "audio/amr",
|
||||
@@ -936,11 +932,6 @@ async def resolve_image_ref_to_base64_data(
|
||||
)
|
||||
|
||||
|
||||
def _image_convert_cache_dir() -> Path:
|
||||
# Reading and encoding must work even when the optional cache is unavailable.
|
||||
return Path(get_astrbot_temp_path()) / CONVERT_CACHE_DIR_NAME
|
||||
|
||||
|
||||
def is_recoverable_image_error(error: Exception) -> bool:
|
||||
"""Identify ordinary input, decoder, network and cache failures.
|
||||
|
||||
@@ -971,55 +962,8 @@ def is_recoverable_image_error(error: Exception) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _image_convert_cache_key(source_bytes: bytes, params: str) -> str:
|
||||
"""Build a content-addressed key for derived image bytes.
|
||||
|
||||
Args:
|
||||
source_bytes: Original encoded image content.
|
||||
params: Algorithm version, output category and effective size.
|
||||
|
||||
Returns:
|
||||
A filesystem-safe content and parameter digest.
|
||||
"""
|
||||
source_digest = hashlib.sha256(source_bytes).hexdigest()[:32]
|
||||
params_digest = hashlib.sha256(params.encode()).hexdigest()[:8]
|
||||
return f"{source_digest}_{params_digest}"
|
||||
|
||||
|
||||
def _image_has_alpha(image: PILImage.Image) -> bool:
|
||||
return "A" in image.getbands() or "transparency" in image.info
|
||||
|
||||
|
||||
def _inspect_image(image_bytes: bytes) -> int:
|
||||
"""Inspect decodable image bytes.
|
||||
|
||||
Args:
|
||||
image_bytes: Encoded image bytes.
|
||||
|
||||
Returns:
|
||||
Number of frames in the validated image.
|
||||
|
||||
Raises:
|
||||
Exception: Raised by Pillow when the bytes are not a decodable image.
|
||||
"""
|
||||
with PILImage.open(io.BytesIO(image_bytes)) as image:
|
||||
frame_count = getattr(image, "n_frames", 1)
|
||||
image.verify()
|
||||
if frame_count == 1:
|
||||
# verify() checks structure, not pixels. Reopen before loading; compatible
|
||||
# still images otherwise bypass every operation that decodes their data.
|
||||
with PILImage.open(io.BytesIO(image_bytes)) as image:
|
||||
image.load()
|
||||
# Animation frames are decoded while building the montage.
|
||||
return frame_count
|
||||
|
||||
|
||||
_IMAGE_CONVERT_CACHE_VERSION = "v9-icc"
|
||||
"""Bump when conversion output semantics change (modes, transparency, sizing)
|
||||
so stale cache entries produced by older code are never served."""
|
||||
|
||||
MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES = 1024 * 1024
|
||||
"""PNG outputs larger than this are flattened onto white and re-encoded as JPEG."""
|
||||
MODEL_IMAGE_MAX_BYTES = 512 * 1024
|
||||
"""Model input image files must be strictly smaller than this limit."""
|
||||
|
||||
|
||||
def normalize_model_image_max_size(value: object) -> int:
|
||||
@@ -1066,195 +1010,104 @@ def normalize_model_image_max_size(value: object) -> int:
|
||||
def _encode_image_frame_bytes(
|
||||
image: PILImage.Image,
|
||||
max_size: int | None = None,
|
||||
quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY,
|
||||
) -> bytes:
|
||||
"""Encode a display-oriented frame as JPEG, or PNG when it carries transparency.
|
||||
"""Encode an owned frame below the byte limit while preserving transparency.
|
||||
|
||||
Args:
|
||||
image: Opened source frame, which is never mutated.
|
||||
image: Decoded frame owned by the caller; its pixels and metadata may change.
|
||||
max_size: Optional longest-edge limit; smaller images are not enlarged.
|
||||
quality: JPEG output quality in the range 1-100.
|
||||
|
||||
Returns:
|
||||
Encoded single-frame JPEG or PNG bytes. A PNG larger than
|
||||
MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES is flattened onto white and
|
||||
re-encoded as JPEG to bound the request payload size. High-bit-depth
|
||||
samples are min-max normalized to 8-bit before JPEG encoding.
|
||||
JPEG or transparent PNG bytes strictly below MODEL_IMAGE_MAX_BYTES.
|
||||
|
||||
Raises:
|
||||
ValueError: The image cannot fit the byte limit even at one pixel.
|
||||
"""
|
||||
oriented = ImageOps.exif_transpose(image)
|
||||
prepared = oriented
|
||||
extras: list[PILImage.Image] = []
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
has_alpha = "A" in image.getbands() or "transparency" in image.info
|
||||
icc_profile = image.info.get("icc_profile")
|
||||
if image.mode not in {"RGB", "RGBA", "L", "LA", "P", "1", "I", "I;16"}:
|
||||
# A source profile is invalid after a color-space conversion such as CMYK.
|
||||
icc_profile = None
|
||||
prepared = image
|
||||
try:
|
||||
has_alpha = _image_has_alpha(oriented)
|
||||
if has_alpha:
|
||||
# Promote RGB/L color-key transparency as well as palette alpha.
|
||||
prepared = oriented.convert("RGBA")
|
||||
elif oriented.mode not in {"1", "L", "LA", "P", "RGB", "RGBA", "I", "I;16"}:
|
||||
prepared = oriented.convert("RGB")
|
||||
if max_size is not None and max(prepared.size) > max_size:
|
||||
# Pillow forces nearest-neighbor resampling for palette and binary
|
||||
# images, regardless of the requested filter.
|
||||
if prepared.mode in {"P", "1"}:
|
||||
prepared = prepared.convert("RGB" if prepared.mode == "P" else "L")
|
||||
# Pillow's integer pre-reduction does not support I;16, although
|
||||
# direct Lanczos resampling does. Keep the original bit depth.
|
||||
if image.mode != "RGBA":
|
||||
prepared = image.convert("RGBA")
|
||||
elif image.mode not in {"RGB", "L", "I", "I;16"}:
|
||||
prepared = image.convert("L" if image.mode == "1" else "RGB")
|
||||
if max_size is not None:
|
||||
prepared.thumbnail(
|
||||
(max_size, max_size),
|
||||
PILImage.Resampling.LANCZOS,
|
||||
reducing_gap=None if prepared.mode == "I;16" else 2.0,
|
||||
)
|
||||
# JPEG saving does not auto-embed the source ICC profile like PNG does;
|
||||
# attach it explicitly, but only when the color space survived intact
|
||||
# (conversions like CMYK -> RGB invalidate the source profile).
|
||||
icc_profile = image.info.get("icc_profile")
|
||||
if image.mode not in {"RGB", "RGBA", "L", "LA", "P", "1", "I", "I;16"}:
|
||||
icc_profile = None
|
||||
save_kwargs = {"icc_profile": icc_profile} if icc_profile else {}
|
||||
if has_alpha:
|
||||
buffer = io.BytesIO()
|
||||
prepared.save(buffer, "PNG", **save_kwargs)
|
||||
data = buffer.getvalue()
|
||||
if len(data) <= MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES:
|
||||
return data
|
||||
# Oversized PNG flattens onto white and re-encodes as JPEG.
|
||||
rgba = prepared if prepared.mode == "RGBA" else prepared.convert("RGBA")
|
||||
extras.append(rgba)
|
||||
flattened = PILImage.new("RGB", rgba.size, (255, 255, 255))
|
||||
extras.append(flattened)
|
||||
flattened.paste(rgba, mask=rgba.getchannel("A"))
|
||||
prepared = flattened
|
||||
elif prepared.mode in {"I", "I;16"}:
|
||||
# convert() clips high-bit-depth samples at 255 instead of scaling;
|
||||
# min-max normalize to 8-bit so the content survives JPEG.
|
||||
if prepared.mode in {"I", "I;16"}:
|
||||
# Normalize high-bit-depth pixels instead of clipping them to 255.
|
||||
low, high = prepared.getextrema()
|
||||
if high > low:
|
||||
scaled = prepared.point(lambda v: (v - low) * (255.0 / (high - low)))
|
||||
extras.append(scaled)
|
||||
prepared = scaled
|
||||
prepared = prepared.convert("L")
|
||||
elif prepared.mode not in {"RGB", "L"}:
|
||||
# JPEG output supports only RGB and L among the remaining modes.
|
||||
prepared = prepared.convert("L" if prepared.mode == "1" else "RGB")
|
||||
buffer = io.BytesIO()
|
||||
prepared.save(buffer, "JPEG", quality=quality, optimize=True, **save_kwargs)
|
||||
return buffer.getvalue()
|
||||
with prepared.point(
|
||||
lambda v: (v - low) * (255.0 / (high - low))
|
||||
) as scaled:
|
||||
prepared = scaled.convert("L")
|
||||
else:
|
||||
prepared = prepared.convert("L")
|
||||
# Do not carry EXIF, text chunks, or other unbounded metadata into previews.
|
||||
prepared.info.clear()
|
||||
save_kwargs = {"icc_profile": icc_profile} if icc_profile else {}
|
||||
while True:
|
||||
with io.BytesIO() as buffer:
|
||||
if has_alpha:
|
||||
prepared.save(buffer, "PNG", **save_kwargs)
|
||||
else:
|
||||
prepared.save(
|
||||
buffer, "JPEG", quality=85, optimize=True, **save_kwargs
|
||||
)
|
||||
data = buffer.getvalue()
|
||||
if len(data) < MODEL_IMAGE_MAX_BYTES:
|
||||
return data
|
||||
if save_kwargs:
|
||||
# Oversized profiles must not defeat the byte limit.
|
||||
save_kwargs.clear()
|
||||
continue
|
||||
if prepared.size == (1, 1):
|
||||
raise ValueError("Image cannot fit the model input byte limit")
|
||||
scale = min(0.85, math.sqrt((MODEL_IMAGE_MAX_BYTES - 1) / len(data)) * 0.95)
|
||||
prepared.thumbnail(
|
||||
(
|
||||
max(1, int(prepared.width * scale)),
|
||||
max(1, int(prepared.height * scale)),
|
||||
),
|
||||
PILImage.Resampling.LANCZOS,
|
||||
)
|
||||
finally:
|
||||
for extra in extras:
|
||||
extra.close()
|
||||
if prepared is not oriented and prepared not in extras:
|
||||
if prepared is not image:
|
||||
prepared.close()
|
||||
oriented.close()
|
||||
|
||||
|
||||
def _read_valid_cached_image_bytes(output_path: Path) -> bytes | None:
|
||||
"""Read a cache entry, treating any miss or corruption as absent.
|
||||
|
||||
The temp cleaner may delete cache entries concurrently: a missing,
|
||||
unreadable, empty, or undecodable entry is treated as a cache miss and
|
||||
the caller rebuilds from the source bytes it already holds.
|
||||
def _prepare_model_image_sync(source_bytes: bytes, max_size: int) -> tuple[bytes, bool]:
|
||||
"""Prepare a preview using a single opened source image.
|
||||
|
||||
Args:
|
||||
output_path: Candidate cache file path.
|
||||
source_bytes: Original encoded image content.
|
||||
max_size: Longest-edge limit for still images and animation montages.
|
||||
|
||||
Returns:
|
||||
Decodable cached bytes, or ``None`` when the entry is unusable.
|
||||
"""
|
||||
try:
|
||||
data = output_path.read_bytes()
|
||||
except OSError as exc:
|
||||
if not is_recoverable_image_error(exc):
|
||||
raise
|
||||
return None
|
||||
if not data:
|
||||
return None
|
||||
try:
|
||||
frame_count = _inspect_image(data)
|
||||
with PILImage.open(io.BytesIO(data)) as image:
|
||||
if (
|
||||
image.format not in {"PNG", "JPEG"}
|
||||
or frame_count != 1
|
||||
or image.getexif().get(274, 1) != 1
|
||||
):
|
||||
return None
|
||||
except Exception as exc:
|
||||
if not is_recoverable_image_error(exc):
|
||||
raise
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def _publish_image_cache_atomic(
|
||||
output_path: Path,
|
||||
data: bytes,
|
||||
) -> None:
|
||||
"""Publish converted bytes via a unique temp file and atomic replace.
|
||||
|
||||
Concurrent readers never observe a partially written cache file. Cache
|
||||
publication failures (e.g. the cache directory vanished mid-write) are
|
||||
non-fatal: callers already hold the complete encoded bytes.
|
||||
|
||||
Args:
|
||||
output_path: Final cache entry path.
|
||||
data: Complete encoded bytes to publish.
|
||||
"""
|
||||
try:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_name = tempfile.mkstemp(dir=output_path.parent, suffix=".tmp")
|
||||
except OSError as exc:
|
||||
if not is_recoverable_image_error(exc):
|
||||
raise
|
||||
logger.debug("Failed to create image cache temp file: %s", exc)
|
||||
return
|
||||
tmp_path = Path(tmp_name)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as temp_file:
|
||||
temp_file.write(data)
|
||||
os.replace(tmp_path, output_path)
|
||||
except OSError as exc:
|
||||
if not is_recoverable_image_error(exc):
|
||||
raise
|
||||
logger.debug("Failed to publish image cache %s: %s", output_path, exc)
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except OSError as exc:
|
||||
if not is_recoverable_image_error(exc):
|
||||
raise
|
||||
logger.debug("Failed to clean up image cache temp file: %s", exc)
|
||||
|
||||
|
||||
def _convert_image_bytes_sync(
|
||||
source_bytes: bytes, max_size: int, quality: int
|
||||
) -> bytes:
|
||||
"""Normalize a validated still image with an optional derived cache.
|
||||
|
||||
Args:
|
||||
source_bytes: Encoded source bytes already checked by _inspect_image.
|
||||
max_size: Longest-edge limit in pixels.
|
||||
quality: JPEG output quality in the range 1-100.
|
||||
|
||||
Returns:
|
||||
Single-frame JPEG or PNG bytes. An oriented JPEG or PNG within the size
|
||||
limit is reused unchanged; anything else is re-encoded.
|
||||
Encoded image bytes and whether they represent an animation montage.
|
||||
"""
|
||||
with PILImage.open(io.BytesIO(source_bytes)) as image:
|
||||
if getattr(image, "n_frames", 1) > 1:
|
||||
return _extract_animation_montage_sync(image, max_size), True
|
||||
if (
|
||||
image.format in {"PNG", "JPEG"}
|
||||
and image.getexif().get(274, 1) == 1
|
||||
and len(source_bytes) < MODEL_IMAGE_MAX_BYTES
|
||||
and max(image.size) <= max_size
|
||||
and image.getexif().get(274, 1) == 1
|
||||
):
|
||||
return source_bytes
|
||||
cache_key = _image_convert_cache_key(
|
||||
source_bytes,
|
||||
f"{_IMAGE_CONVERT_CACHE_VERSION}|convert|s={max_size}|q={quality}",
|
||||
)
|
||||
output_path = _image_convert_cache_dir() / (cache_key + ".img")
|
||||
cached = _read_valid_cached_image_bytes(output_path)
|
||||
if cached is not None:
|
||||
return cached
|
||||
encoded = _encode_image_frame_bytes(image, max_size=max_size, quality=quality)
|
||||
_publish_image_cache_atomic(output_path, encoded)
|
||||
return encoded
|
||||
# Validate even byte-identical passthroughs without decoding twice.
|
||||
image.load()
|
||||
return source_bytes, False
|
||||
return _encode_image_frame_bytes(image, max_size), False
|
||||
|
||||
|
||||
def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]:
|
||||
@@ -1273,91 +1126,70 @@ def _even_frame_indices(total_frames: int, max_frames: int) -> list[int]:
|
||||
return sorted({round(i * (total_frames - 1) / (count - 1)) for i in range(count)})
|
||||
|
||||
|
||||
def _extract_animation_montage_sync(
|
||||
source_bytes: bytes,
|
||||
max_size: int,
|
||||
quality: int,
|
||||
) -> tuple[bytes, bool]:
|
||||
"""Tile evenly spaced frames of an animated image into one grid montage.
|
||||
|
||||
Frame sampling includes the first and last frames; grid cells beyond the
|
||||
available frames stay blank (white). The montage is flattened onto a white
|
||||
background and its longest edge is capped at ``max_size`` (never upscaled).
|
||||
def _extract_animation_montage_sync(image: PILImage.Image, max_size: int) -> bytes:
|
||||
"""Tile evenly spaced animation frames into one white 3x3 preview.
|
||||
|
||||
Args:
|
||||
source_bytes: Encoded animated image bytes.
|
||||
max_size: Longest edge of the montage in pixels.
|
||||
image: Opened animation owned by the caller.
|
||||
max_size: Longest-edge limit of the montage, without upscaling.
|
||||
|
||||
Returns:
|
||||
Tuple of the montage bytes and whether extraction actually ran
|
||||
(``False`` when a valid cached montage was served).
|
||||
JPEG bytes below the byte limit, with unused cells left white.
|
||||
"""
|
||||
cache_key = _image_convert_cache_key(
|
||||
source_bytes,
|
||||
f"{_IMAGE_CONVERT_CACHE_VERSION}|montage|s={max_size}|q={quality}",
|
||||
total_frames = getattr(image, "n_frames", 1)
|
||||
# APNG's independent default image is a cover, not an animation frame.
|
||||
first_frame = 1 if image.info.get("default_image", False) else 0
|
||||
frame_indices = [
|
||||
first_frame + index
|
||||
for index in _even_frame_indices(
|
||||
total_frames - first_frame, ANIMATED_MONTAGE_FRAME_COUNT
|
||||
)
|
||||
]
|
||||
image.seek(first_frame)
|
||||
with ImageOps.exif_transpose(image) as oriented:
|
||||
display_size = oriented.size
|
||||
# Floor the per-cell scale so the montage never exceeds max_size.
|
||||
longest_edge = max(display_size) * ANIMATED_MONTAGE_GRID
|
||||
scale = min(1.0, max(max_size, 1) / longest_edge)
|
||||
cell_size = (
|
||||
max(1, int(display_size[0] * scale)),
|
||||
max(1, int(display_size[1] * scale)),
|
||||
)
|
||||
output_path = _image_convert_cache_dir() / (cache_key + ".img")
|
||||
cached = _read_valid_cached_image_bytes(output_path)
|
||||
if cached is not None:
|
||||
return cached, False
|
||||
with PILImage.open(io.BytesIO(source_bytes)) as image:
|
||||
total_frames = getattr(image, "n_frames", 1)
|
||||
# APNG's independent default image is a cover, not an animation frame.
|
||||
first_frame = 1 if image.info.get("default_image", False) else 0
|
||||
frame_indices = [
|
||||
first_frame + index
|
||||
for index in _even_frame_indices(
|
||||
total_frames - first_frame, ANIMATED_MONTAGE_FRAME_COUNT
|
||||
)
|
||||
]
|
||||
image.seek(first_frame)
|
||||
with ImageOps.exif_transpose(image) as oriented:
|
||||
display_size = oriented.size
|
||||
# Floor the per-cell scale so the montage never exceeds max_size.
|
||||
longest_edge = max(display_size) * ANIMATED_MONTAGE_GRID
|
||||
scale = min(1.0, max(max_size, 1) / longest_edge)
|
||||
cell_size = (
|
||||
max(1, int(display_size[0] * scale)),
|
||||
max(1, int(display_size[1] * scale)),
|
||||
)
|
||||
canvas = PILImage.new(
|
||||
"RGB",
|
||||
(
|
||||
cell_size[0] * ANIMATED_MONTAGE_GRID,
|
||||
cell_size[1] * ANIMATED_MONTAGE_GRID,
|
||||
),
|
||||
(255, 255, 255),
|
||||
)
|
||||
try:
|
||||
for out_index, frame_index in enumerate(frame_indices):
|
||||
image.seek(frame_index)
|
||||
with (
|
||||
ImageOps.exif_transpose(image) as oriented,
|
||||
oriented.convert("RGBA") as frame,
|
||||
):
|
||||
resized = frame
|
||||
try:
|
||||
if frame.size != cell_size:
|
||||
resized = frame.resize(
|
||||
cell_size, PILImage.Resampling.LANCZOS
|
||||
)
|
||||
# Alpha shows the white canvas through transparent pixels.
|
||||
canvas.paste(
|
||||
resized,
|
||||
(
|
||||
(out_index % ANIMATED_MONTAGE_GRID) * cell_size[0],
|
||||
(out_index // ANIMATED_MONTAGE_GRID) * cell_size[1],
|
||||
),
|
||||
resized,
|
||||
)
|
||||
finally:
|
||||
if resized is not frame:
|
||||
resized.close()
|
||||
encoded = _encode_image_frame_bytes(canvas, quality=quality)
|
||||
finally:
|
||||
canvas.close()
|
||||
_publish_image_cache_atomic(output_path, encoded)
|
||||
return encoded, True
|
||||
canvas = PILImage.new(
|
||||
"RGB",
|
||||
(
|
||||
cell_size[0] * ANIMATED_MONTAGE_GRID,
|
||||
cell_size[1] * ANIMATED_MONTAGE_GRID,
|
||||
),
|
||||
(255, 255, 255),
|
||||
)
|
||||
try:
|
||||
for out_index, frame_index in enumerate(frame_indices):
|
||||
image.seek(frame_index)
|
||||
with (
|
||||
ImageOps.exif_transpose(image) as oriented,
|
||||
oriented.convert("RGBA") as frame,
|
||||
):
|
||||
resized = frame
|
||||
try:
|
||||
if frame.size != cell_size:
|
||||
resized = frame.resize(cell_size, PILImage.Resampling.LANCZOS)
|
||||
# Alpha shows the white canvas through transparent pixels.
|
||||
canvas.paste(
|
||||
resized,
|
||||
(
|
||||
(out_index % ANIMATED_MONTAGE_GRID) * cell_size[0],
|
||||
(out_index // ANIMATED_MONTAGE_GRID) * cell_size[1],
|
||||
),
|
||||
resized,
|
||||
)
|
||||
finally:
|
||||
if resized is not frame:
|
||||
resized.close()
|
||||
encoded = _encode_image_frame_bytes(canvas)
|
||||
finally:
|
||||
canvas.close()
|
||||
return encoded
|
||||
|
||||
|
||||
async def prepare_model_image(
|
||||
@@ -1365,44 +1197,25 @@ async def prepare_model_image(
|
||||
*,
|
||||
max_size: int,
|
||||
output_dir: Path,
|
||||
quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY,
|
||||
montage_max_size: int | None = None,
|
||||
) -> tuple[str, bool] | None:
|
||||
"""Prepare a single local model-ready image for the caller to own until consumption.
|
||||
|
||||
Args:
|
||||
image_ref: Source reference accepted by MediaResolver.
|
||||
max_size: Validated longest-edge limit for stills.
|
||||
output_dir: Directory for independent request-owned working files.
|
||||
quality: JPEG output quality in the range 1-100.
|
||||
montage_max_size: Optional longest-edge limit for animation montages.
|
||||
CUA sessions lift the still-image cap to keep pixel coordinates 1:1,
|
||||
but montages are never used for coordinates, so callers pass the
|
||||
configured limit here to keep the 3x3 canvas bounded. Defaults to
|
||||
``max_size``.
|
||||
max_size: Longest-edge limit for both still images and montages.
|
||||
output_dir: Directory for event-owned working files.
|
||||
|
||||
Returns:
|
||||
Tuple of an existing JPEG or PNG path and whether it is an animation
|
||||
frame montage (including cache hits), or None for a recoverable input
|
||||
or write failure.
|
||||
The caller owns this file; shared cache entries are never returned.
|
||||
A preview path and whether it is an animation montage, or None for a
|
||||
recoverable input or write failure. The caller must delete the preview
|
||||
after use. The original image is never overwritten.
|
||||
"""
|
||||
try:
|
||||
async with MediaResolver(image_ref, media_type="image").as_path() as source:
|
||||
image_bytes = await asyncio.to_thread(source.read_bytes)
|
||||
frame_count = await asyncio.to_thread(_inspect_image, image_bytes)
|
||||
is_montage = frame_count > 1
|
||||
if is_montage:
|
||||
converted_bytes, _ = await asyncio.to_thread(
|
||||
_extract_animation_montage_sync,
|
||||
image_bytes,
|
||||
montage_max_size if montage_max_size is not None else max_size,
|
||||
quality,
|
||||
)
|
||||
else:
|
||||
converted_bytes = await asyncio.to_thread(
|
||||
_convert_image_bytes_sync, image_bytes, max_size, quality
|
||||
)
|
||||
converted_bytes, is_montage = await asyncio.to_thread(
|
||||
_prepare_model_image_sync, image_bytes, max_size
|
||||
)
|
||||
# Publish the working file synchronously after encoding, so cancellation
|
||||
# cannot leave an untracked background write alive after this call.
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# Unreleased
|
||||
|
||||
- Local Agent input images are always prepared as JPEG/PNG files strictly below 512 KiB. Transparent previews retain PNG alpha; animations become 3×3 montages. Original attachment paths remain available to tools, and event-owned previews are deleted after use without a shared conversion cache.
|
||||
- Configuration mapping: **Enable image compression** (`provider_settings.image_compress_enabled`) is replaced by always-on preparation; **JPEG quality** (`provider_settings.image_compress_options.quality`) is replaced by automatic size control; **Maximum edge length** (`provider_settings.image_compress_options.max_size`) remains available. User attachments in CUA sessions follow the same limits.
|
||||
@@ -481,20 +481,12 @@
|
||||
"description": "User Prompt",
|
||||
"hint": "You can use {{prompt}} as a placeholder for user input. If no placeholder is provided, it will be added before the user input."
|
||||
},
|
||||
"image_compress_enabled": {
|
||||
"description": "Enable image compression",
|
||||
"hint": "Enabled by default. Images are resized or converted as needed before being sent to multimodal models; compliant JPEG/PNG images are sent unchanged, and animations become a preview grid."
|
||||
},
|
||||
"image_compress_options": {
|
||||
"description": "Image compression settings",
|
||||
"hint": "Control the size limit and JPEG output quality for model input images.",
|
||||
"description": "Model input image settings",
|
||||
"hint": "Images are always prepared as JPEG/PNG below 512 KiB. Original attachments are preserved.",
|
||||
"max_size": {
|
||||
"description": "Maximum edge length",
|
||||
"hint": "Longest edge of the compressed image in pixels. Images larger than this are resized proportionally."
|
||||
},
|
||||
"quality": {
|
||||
"description": "JPEG quality",
|
||||
"hint": "JPEG output quality from 1 to 100. Higher values preserve more detail but produce larger files."
|
||||
"hint": "Longest edge sent to the model, in pixels. Images may be reduced further to stay below 512 KiB; original attachment paths are unchanged."
|
||||
}
|
||||
},
|
||||
"reachability_check": {
|
||||
|
||||
@@ -582,20 +582,12 @@
|
||||
"description": "ユーザープロンプト",
|
||||
"hint": "{{prompt}} をユーザー入力のプレースホルダーとして使用できます。プレースホルダーを指定しない場合は、ユーザー入力の前に追加されます。"
|
||||
},
|
||||
"image_compress_enabled": {
|
||||
"description": "画像圧縮を有効化",
|
||||
"hint": "デフォルトで有効。マルチモーダルモデルへの送信前に画像を必要に応じて圧縮・変換します。条件を満たす JPEG/PNG はそのまま送信され、アニメーションはプレビューグリッドになります。"
|
||||
},
|
||||
"image_compress_options": {
|
||||
"description": "画像圧縮設定",
|
||||
"hint": "モデルへ送信する画像のサイズ上限と JPEG 出力品質を設定します。",
|
||||
"description": "モデル入力画像の設定",
|
||||
"hint": "画像は常に 512 KiB 未満の JPEG/PNG に処理されます。元の添付ファイルは保持されます。",
|
||||
"max_size": {
|
||||
"description": "最大辺長",
|
||||
"hint": "圧縮後の画像の長辺をピクセル単位で指定します。このサイズを超える場合は縦横比を維持して縮小します。"
|
||||
},
|
||||
"quality": {
|
||||
"description": "JPEG 品質",
|
||||
"hint": "JPEG の出力品質を 1~100 の範囲で指定します。値が高いほど画質が向上し、ファイルサイズも大きくなります。"
|
||||
"hint": "モデルに送信する画像の最長辺(ピクセル)。512 KiB 未満にするため、さらに縮小される場合があります。元の添付ファイルのパスは変わりません。"
|
||||
}
|
||||
},
|
||||
"reachability_check": {
|
||||
|
||||
@@ -621,20 +621,12 @@
|
||||
"description": "Промпт пользователя",
|
||||
"hint": "Вы можете использовать {{prompt}} как заполнитель для ввода. Если заполнитель не указан, он будет добавлен перед текстом пользователя."
|
||||
},
|
||||
"image_compress_enabled": {
|
||||
"description": "Включить сжатие изображений",
|
||||
"hint": "Включено по умолчанию. Перед отправкой мультимодальным моделям изображения при необходимости сжимаются или преобразуются; корректные JPEG/PNG отправляются без изменений, анимации собираются в сетку предпросмотра."
|
||||
},
|
||||
"image_compress_options": {
|
||||
"description": "Настройки сжатия изображений",
|
||||
"hint": "Управляет ограничением размера изображений для модели и качеством JPEG.",
|
||||
"description": "Настройки изображений для модели",
|
||||
"hint": "Изображения всегда преобразуются в JPEG/PNG размером менее 512 КиБ. Исходные вложения сохраняются.",
|
||||
"max_size": {
|
||||
"description": "Максимальная длина стороны",
|
||||
"hint": "Максимальная длина стороны сжатого изображения в пикселях. Более крупные изображения пропорционально уменьшаются."
|
||||
},
|
||||
"quality": {
|
||||
"description": "Качество JPEG",
|
||||
"hint": "Качество JPEG от 1 до 100. Более высокие значения сохраняют больше деталей, но увеличивают размер файла."
|
||||
"hint": "Максимальная длина стороны изображения для модели в пикселях. При необходимости изображение уменьшается дополнительно до размера менее 512 КиБ. Путь к исходному вложению не меняется."
|
||||
}
|
||||
},
|
||||
"reachability_check": {
|
||||
|
||||
@@ -502,20 +502,12 @@
|
||||
"description": "用户提示词",
|
||||
"hint": "可使用 {{prompt}} 作为用户输入的占位符。如果不输入占位符则代表添加在用户输入的前面。"
|
||||
},
|
||||
"image_compress_enabled": {
|
||||
"description": "启用图片压缩",
|
||||
"hint": "默认开启。发送给多模态模型前按需压缩转换图片:合规的 JPEG/PNG 原样发送,动图生成拼图预览。"
|
||||
},
|
||||
"image_compress_options": {
|
||||
"description": "图片压缩配置",
|
||||
"hint": "用于控制发送给模型的图片尺寸上限和 JPEG 输出质量。",
|
||||
"description": "模型输入图片设置",
|
||||
"hint": "图片始终处理为严格小于 512 KiB 的 JPEG/PNG,原始附件保留。",
|
||||
"max_size": {
|
||||
"description": "最大边长",
|
||||
"hint": "压缩后图片的最长边,单位为像素。超过该尺寸时会按比例缩放。"
|
||||
},
|
||||
"quality": {
|
||||
"description": "JPEG 质量",
|
||||
"hint": "JPEG 输出质量,范围为 1-100。值越高,画质越好,文件也越大。"
|
||||
"hint": "发送给模型的图片最长边,单位为像素。为保证单张图片小于 512 KiB,必要时会进一步缩小;原始附件路径保持不变。"
|
||||
}
|
||||
},
|
||||
"reachability_check": {
|
||||
|
||||
@@ -1,28 +1,35 @@
|
||||
# Model input images in the local ProcessStage
|
||||
# Model input images
|
||||
|
||||
“Enable image compression” (`provider_settings.image_compress_enabled`) is enabled by default. The local Agent branch of ProcessStage prepares the current input before building the Agent, including ordinary attachments, quoted images and plugin `ProviderRequest` images. It checks images added or replaced by `OnLLMRequestEvent` once more before Runner reset.
|
||||
The local Agent always prepares current input images before sending them to the model. This covers ordinary attachments, quoted images, and plugin `ProviderRequest` images, including images added or replaced by `OnLLMRequestEvent`.
|
||||
|
||||
## Preparation rules
|
||||
## Limits and formats
|
||||
|
||||
- Compliant still images are sent unchanged: already JPEG or PNG, correctly oriented, and within the size limit.
|
||||
- Other still images are orientation-corrected, resized and re-encoded as needed. Opaque images become JPEG, controlled by `image_compress_options.quality` (default 95), with high-bit-depth samples normalized to 8-bit. Images with transparency become PNG; a PNG larger than 1 MB is flattened onto a white background and re-encoded as JPEG to bound the payload size.
|
||||
- Animations are detected from their frames, including GIF, animated WebP and APNG. Up to nine evenly sampled frames, including the first and last, become one white 3×3 grid. Unused cells stay white; an APNG independent cover is excluded.
|
||||
- Stills and montages share `image_compress_options.max_size` (default 1280). Small images are not enlarged.
|
||||
- Disabling compression keeps generic localization and reading, without resizing, transcoding, sampling or consulting the derived-image cache.
|
||||
- Each prepared image file is **strictly smaller than 512 KiB** (524,288 bytes). This limit applies to image bytes before Base64 encoding, not the entire request.
|
||||
- **Maximum edge length**, under **Configuration**, remains configurable through `provider_settings.image_compress_options.max_size` (default 1280 pixels). Images keep their aspect ratio and are never enlarged. They may be reduced further to meet the byte limit.
|
||||
- Correctly oriented JPEG/PNG images already within both limits retain their original bytes. Other still images are orientation-corrected and encoded as JPEG, or PNG when they have transparency. Transparent images retain their alpha channel even when further reduction is necessary.
|
||||
- GIF, animated WebP, and APNG inputs become a white 3×3 frame montage. Up to nine evenly spaced frames include the first and last animation frames; unused cells stay blank. An independent APNG cover is excluded.
|
||||
- Unneeded metadata is removed when re-encoding. A color profile is retained when compatible with the output and the byte limit.
|
||||
|
||||
These limits also apply to user attachments in CUA sessions. Screenshot and other tool-result images keep their own handling.
|
||||
|
||||
> [!TIP]
|
||||
> When the computer-use runtime is `sandbox` and the sandbox booter is `cua`, input images are not resized, so pixel coordinates read by coordinate-based tools stay 1:1. Compliant images pass through byte-exact (no lossy re-encoding, avoiding JPEG color shifts); format conversion for other formats and animation montages still apply. Images above roughly 5 MB may exceed provider image upload limits and trigger a warning in the logs.
|
||||
The Agent receives readable local paths. Original image files and event components keep their original content, and attachment text continues to reference the source image. Providers only read/encode references and assemble their protocols.
|
||||
## Configuration changes
|
||||
|
||||
## Errors and lifetime
|
||||
| Previous setting | Current behavior |
|
||||
| --- | --- |
|
||||
| Enable image compression (`image_compress_enabled`) | Removed; current input images are always prepared. |
|
||||
| JPEG quality (`image_compress_options.quality`) | Removed; output size is controlled automatically. |
|
||||
| Maximum edge length (`image_compress_options.max_size`) | Retained; applies together with the strict 512 KiB limit. |
|
||||
|
||||
An unreadable, corrupt or locally undecodable image is skipped with a short warning; valid text and other images remain. If nothing usable remains, request preparation supplies a placeholder. Cancellation, resource exhaustion and programming errors are not treated as bad images.
|
||||
Values of the removed settings in existing configuration files no longer affect image preparation.
|
||||
|
||||
Derived bytes are cached by source content, effective size, JPEG quality, still/montage category and algorithm version. Missing or corrupt entries are rebuilt. Each request receives independent event-owned working files, so event cleanup does not remove shared cache files. An unwritable cache can be bypassed; inability to create the working file skips that image.
|
||||
## Original files and preview lifetime
|
||||
|
||||
Successfully localized event attachments and sources adopted during request collection remain available after event cleanup, including quoted images. Attachment text keeps these source paths; model working copies and request-only downloads are still cleaned up. Newly materialized sources stay event-owned if collection fails or is cancelled. Retained files under the temporary directory remain subject to `temp_dir_max_size` cleanup and are not permanent storage.
|
||||
`image_urls` and image content parts use the prepared preview files. The attachment paths shown to the model for file access continue to point to the **original images**, so tools can read the originals. Original files and event image components are not overwritten.
|
||||
|
||||
Existing message serialization stores the visual content the model received, including prepared images, before temporary paths expire. Replay and fallback reuse that content; old history is not rescanned or migrated.
|
||||
Preview files are owned by the current event and deleted when the event finishes, including failure or cancellation. There is no shared image-conversion cache. Repeated references within one request reuse the same preview, including the preparation pass after the request hook.
|
||||
|
||||
This feature covers only current inputs through the local ProcessStage. Third-party Agent backends, direct plugin calls to Agent/Provider APIs, tool-result images, read_file and CUA retain their existing behavior. Tool-image handling is a separate follow-up and is not required for this feature.
|
||||
Successfully localized original attachments, including quoted images, survive event cleanup. Retained originals under the temporary directory remain subject to `temp_dir_max_size` cleanup and are not permanent storage. Sources that have not yet been successfully adopted when request collection fails remain event-owned.
|
||||
|
||||
Conversation history captures the prepared image content as Base64 data URIs before preview files expire. Old history is not reprocessed. An unreadable image is skipped; valid text and other images remain. Cancellation, resource exhaustion, and programming errors are not treated as bad images.
|
||||
|
||||
Third-party Agent backends, direct calls that bypass the local pipeline, and tool-result images are outside this input preparation flow.
|
||||
|
||||
@@ -1,29 +1,35 @@
|
||||
# 图片处理
|
||||
|
||||
使用内置 Agent 执行器处理新消息时,AstrBot 默认会在发送图片给模型前按需处理图片,减少格式不兼容的问题。处理不会覆盖原图。
|
||||
使用内置 Agent 处理新消息时,AstrBot 始终在发送给模型前处理本次输入的图片,包括普通附件、引用图片,以及插件 `ProviderRequest` 中的图片。请求 Hook 新增或替换的图片也会处理。
|
||||
|
||||
## 配置
|
||||
## 大小限制与格式
|
||||
|
||||
在管理面板的 **配置文件** 页面搜索 **启用图片压缩**。该选项默认开启,关闭后不再缩放、转码或生成动图拼图。
|
||||
- 模型使用的单张图片文件**严格小于 512 KiB**(524,288 字节)。限制针对 Base64 编码前的图片文件,不是整个请求体。
|
||||
- 在管理面板的**配置文件**页面搜索**最大边长**,对应 `provider_settings.image_compress_options.max_size`,默认 1280 像素。图片保持宽高比,小图不会放大;为满足文件体积限制,必要时会进一步缩小。
|
||||
- 方向正常、像素尺寸及文件体积均合规的 JPEG/PNG 保留原始字节。其余静图修正方向后编码为 JPEG;含透明度的图片输出 PNG,即使需要进一步缩小也保留透明通道。
|
||||
- GIF、动画 WebP、APNG 等动图均匀选取最多 9 帧,包含首尾动画帧,生成白底 3×3 拼图。不足 9 帧的格子留白,APNG 的独立封面不参与采样。
|
||||
- 重新编码时移除不必要的元数据;与输出色彩空间兼容、且不影响体积达标的色彩配置保留。
|
||||
|
||||
**最大边长**默认为 1280 像素,静图和动图拼图共用此设置,小图不会放大。**压缩质量**(1-100,默认 95)控制 JPEG 输出的画质与体积。
|
||||
CUA 会话中的用户输入图片也遵守这些限制;截图及其他工具返回的图片继续使用各自的处理逻辑。
|
||||
|
||||
## 配置项变化
|
||||
|
||||
> [!TIP]
|
||||
> 当“电脑使用”运行时为沙箱且沙箱 Booter 为 CUA 时,为避免像素坐标漂移,输入图片不按最大边长缩放,合规图片原样发送(不重新编码,避免 JPEG 变色);格式转换与动图拼图等其余处理仍然生效。超过约 5 MB 的大图可能超出服务商的图片体积限制,此时日志会给出警告。
|
||||
## 处理方式
|
||||
| 原配置项 | 现在的行为 |
|
||||
| --- | --- |
|
||||
| 启用图片压缩(`image_compress_enabled`) | 已移除,当前输入图片始终处理。 |
|
||||
| 压缩质量(`image_compress_options.quality`) | 已移除,自动控制输出体积。 |
|
||||
| 最大边长(`image_compress_options.max_size`) | 保留,同时遵守严格小于 512 KiB 的文件体积限制。 |
|
||||
|
||||
- **合规静图原样发送**:已是 JPEG/PNG、方向正常且未超过最大边长的图片不做任何改动。
|
||||
- **其余静图按需转换**:方向异常、尺寸超限或格式不通用的图片会修正方向、缩放后重新编码。无透明度的图片输出 JPEG(16 位等高位深图片会归一化为 8 位);含透明度的图片输出 PNG,若 PNG 超过 1 MB 则扁平化为白底 JPEG,控制发送体积。
|
||||
- **动图**:从 GIF、动画 WebP、APNG 等图片中均匀选取最多 9 帧,包含首尾动画帧,生成白底 3×3 拼图;不足 9 帧的格子留白。单帧 GIF 和静态 WebP 按静图处理。
|
||||
- **无法读取或解码的图片**:跳过该图片,其他文字和正常图片继续处理。
|
||||
旧配置文件中遗留的启用开关和 JPEG 质量值不再影响图片处理。
|
||||
|
||||
> [!TIP]
|
||||
> 如需更小的请求体积,可调低“最大边长”或“压缩质量”。
|
||||
## 原图路径与文件清理
|
||||
|
||||
## 文件保留与清理
|
||||
`image_urls` 及图片内容块使用处理后的预览文件。提供给模型、供工具读取文件的附件说明仍然指向**原始图片路径**,不会替换为预览路径。原始文件和事件中的图片组件均不被覆盖。
|
||||
|
||||
预处理成功本地化的附件原图,以及请求收集成功接管的原图(含引用图片),不会在事件结束时立即删除,附件说明继续引用这些原图路径。模型使用的派生文件和仅用于请求的下载文件仍随事件清理;请求收集失败或取消时,新生成且尚未接管的原图也会清理。临时目录中的保留文件仍受 `temp_dir_max_size` 容量清理约束,不属于永久存储。
|
||||
模型预览文件归当前事件所有,事件完成、失败或取消后清理,不再保留跨请求图片转换缓存。同一请求中重复出现的图片引用复用同一份预览,请求 Hook 前后的准备过程也会去重。
|
||||
|
||||
> [!NOTE]
|
||||
> 本功能仅适用于经过本地消息流水线的本次输入,包括引用图片。工具回图、第三方 Agent 后端及插件绕过流水线直接调用模型不在处理范围内。会话历史保存模型实际看到的图片内容,不会重新转换旧图片。
|
||||
成功本地化的附件原图,包括引用图片,不随事件结束删除。临时目录中的原图仍受 `temp_dir_max_size` 容量清理约束,不属于永久存储。请求收集失败时,尚未成功接管的源文件仍按事件临时文件清理。
|
||||
|
||||
对话历史会在预览文件删除前保存模型使用的图片内容,采用 Base64 data URI;旧历史不会重新处理。无法读取的图片会跳过,不影响其他文字和正常图片;取消、资源耗尽及程序错误不会被当作坏图忽略。
|
||||
|
||||
第三方 Agent 后端、绕过本地流水线的直接调用,以及工具返回的图片,不属于本次输入图片处理范围。
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
import asyncio
|
||||
import errno
|
||||
import random
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
@@ -14,8 +14,8 @@ from astrbot.core.utils.io import DownloadFileHTTPError
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_cache(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(media, "get_astrbot_temp_path", lambda: str(tmp_path / "cache"))
|
||||
def isolated_temp(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(media, "get_astrbot_temp_path", lambda: str(tmp_path / "temp"))
|
||||
|
||||
|
||||
async def _prepare_image_path(*args, **kwargs):
|
||||
@@ -69,16 +69,21 @@ async def test_rotated_jpeg_is_normalized_to_upright_jpeg(tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_oversized_alpha_png_flattens_to_jpeg(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(media, "MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES", 1)
|
||||
source = tmp_path / "alpha.png"
|
||||
Image.new("RGBA", (2000, 100), (255, 0, 0, 128)).save(source)
|
||||
@pytest.mark.parametrize("fmt", ["PNG", "WEBP"])
|
||||
async def test_transparent_images_shrink_without_losing_alpha(tmp_path, fmt):
|
||||
source = tmp_path / f"alpha.{fmt.lower()}"
|
||||
pixels = random.Random(9703).randbytes(1280 * 960 * 3)
|
||||
image = Image.frombytes("RGB", (1280, 960), pixels)
|
||||
image.putalpha(128)
|
||||
image.save(source, fmt, lossless=True)
|
||||
original = source.read_bytes()
|
||||
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
|
||||
assert path and path.endswith(".jpg")
|
||||
with Image.open(path) as image:
|
||||
assert image.format == "JPEG" and image.mode == "RGB"
|
||||
r, g, b = image.getpixel((0, 0))
|
||||
assert r > 200 and 90 < g < 170 and 90 < b < 170
|
||||
assert path and Path(path).stat().st_size < 512 * 1024
|
||||
with Image.open(path) as result:
|
||||
assert result.format == "PNG" and result.mode == "RGBA"
|
||||
assert max(result.size) < 1280
|
||||
assert result.getchannel("A").getextrema() == (128, 128)
|
||||
assert source.read_bytes() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -124,19 +129,49 @@ async def test_cmyk_icc_profile_is_not_attached_to_rgb_jpeg(tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jpeg_quality_is_honored_and_cached_separately(tmp_path):
|
||||
source = tmp_path / "photo.bmp"
|
||||
Image.new("RGB", (200, 100), "red").save(source)
|
||||
low = await _prepare_image_path(
|
||||
str(source), max_size=1280, output_dir=tmp_path, quality=10
|
||||
)
|
||||
high = await _prepare_image_path(
|
||||
str(source), max_size=1280, output_dir=tmp_path, quality=95
|
||||
)
|
||||
assert low and high and low != high
|
||||
low_bytes, high_bytes = Path(low).read_bytes(), Path(high).read_bytes()
|
||||
assert low_bytes.startswith(b"\xff\xd8") and high_bytes.startswith(b"\xff\xd8")
|
||||
assert len(low_bytes) < len(high_bytes)
|
||||
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "WEBP", "BMP"])
|
||||
async def test_opaque_images_fit_the_byte_limit(tmp_path, fmt):
|
||||
source = tmp_path / f"noise.{fmt.lower()}"
|
||||
pixels = random.Random(9703).randbytes(1280 * 960 * 3)
|
||||
Image.frombytes("RGB", (1280, 960), pixels).save(source, fmt, quality=100)
|
||||
original = source.read_bytes()
|
||||
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
|
||||
assert path and Path(path).stat().st_size < 512 * 1024
|
||||
with Image.open(path) as result:
|
||||
assert result.format == "JPEG" and max(result.size) <= 1280
|
||||
assert source.read_bytes() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fmt", ["JPEG", "PNG"])
|
||||
@pytest.mark.parametrize("size", [512 * 1024 - 1, 512 * 1024, 512 * 1024 + 1])
|
||||
async def test_byte_limit_is_strict_even_for_dimension_compliant_images(
|
||||
tmp_path, fmt, size
|
||||
):
|
||||
source = tmp_path / f"small.{fmt.lower()}"
|
||||
Image.new("RGB", (32, 16), "red").save(source, fmt)
|
||||
original = source.read_bytes().ljust(size, b"\0")
|
||||
source.write_bytes(original)
|
||||
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
|
||||
assert path and Path(path).stat().st_size < 512 * 1024
|
||||
assert (Path(path).read_bytes() == original) == (size < 512 * 1024)
|
||||
assert source.read_bytes() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fmt", ["JPEG", "PNG"])
|
||||
async def test_oversized_metadata_cannot_bypass_byte_limit(tmp_path, fmt):
|
||||
source = tmp_path / f"metadata.{fmt.lower()}"
|
||||
image = Image.new("RGBA" if fmt == "PNG" else "RGB", (32, 16), "red")
|
||||
image.save(source, fmt, icc_profile=random.Random(512).randbytes(600 * 1024))
|
||||
original = source.read_bytes()
|
||||
assert len(original) >= 512 * 1024
|
||||
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
|
||||
assert path and Path(path).stat().st_size < 512 * 1024
|
||||
with Image.open(path) as result:
|
||||
assert result.size == (32, 16)
|
||||
assert not result.info.get("icc_profile")
|
||||
assert source.read_bytes() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -273,40 +308,22 @@ def test_size_normalization(value, expected):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("bad_cache", [b"", b"bad", None, "animated"])
|
||||
async def test_bad_cache_rebuilt_and_work_files_independent(
|
||||
tmp_path, monkeypatch, bad_cache
|
||||
):
|
||||
async def test_working_files_are_independent_without_shared_cache(tmp_path):
|
||||
source = tmp_path / "source.bmp"
|
||||
Image.new("RGB", (32, 16), "red").save(source)
|
||||
first = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
|
||||
cache = next((tmp_path / "cache" / media.CONVERT_CACHE_DIR_NAME).glob("*.img"))
|
||||
if bad_cache is None:
|
||||
cache.unlink()
|
||||
elif bad_cache == "animated":
|
||||
Image.new("RGB", (16, 8), "blue").save(
|
||||
cache,
|
||||
"GIF",
|
||||
save_all=True,
|
||||
append_images=[Image.new("RGB", (16, 8), "red")],
|
||||
)
|
||||
else:
|
||||
cache.write_bytes(bad_cache)
|
||||
second = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
|
||||
output = tmp_path / "output"
|
||||
first = await _prepare_image_path(str(source), max_size=16, output_dir=output)
|
||||
second = await _prepare_image_path(str(source), max_size=16, output_dir=output)
|
||||
assert first != second and Path(first).read_bytes() == Path(second).read_bytes()
|
||||
assert set(output.iterdir()) == {Path(first), Path(second)}
|
||||
Path(first).unlink()
|
||||
assert cache.is_file() and Path(second).is_file()
|
||||
encode = Mock(side_effect=AssertionError("valid cache must skip encoding"))
|
||||
monkeypatch.setattr(media, "_encode_image_frame_bytes", encode)
|
||||
third = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
|
||||
assert Path(third).read_bytes() == Path(second).read_bytes()
|
||||
assert Path(second).is_file() and source.is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unwritable_cache_bypassed_but_work_file_failure_skips(tmp_path):
|
||||
async def test_work_file_failure_skips_image(tmp_path):
|
||||
source = tmp_path / "source.bmp"
|
||||
Image.new("RGB", (32, 16), "red").save(source)
|
||||
(tmp_path / "cache").write_text("blocks directory creation")
|
||||
path = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
|
||||
assert Path(path).is_file()
|
||||
assert (
|
||||
@@ -351,20 +368,6 @@ async def test_fatal_image_failures_propagate(tmp_path, monkeypatch, error):
|
||||
await _prepare_image_path("ref", max_size=1280, output_dir=tmp_path)
|
||||
|
||||
|
||||
def test_cache_publication_is_atomic_and_closes_file(tmp_path, monkeypatch):
|
||||
output = tmp_path / "cache.png"
|
||||
original_replace = media.os.replace
|
||||
|
||||
def replace(source, target):
|
||||
assert Path(source).read_bytes() == b"complete"
|
||||
assert not output.exists()
|
||||
original_replace(source, target)
|
||||
|
||||
monkeypatch.setattr(media.os, "replace", replace)
|
||||
media._publish_image_cache_atomic(output, b"complete")
|
||||
assert output.read_bytes() == b"complete" and not list(tmp_path.glob("*.tmp"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [403, 404, 503])
|
||||
async def test_real_download_status_failure_is_skipped_without_secret_logs(
|
||||
@@ -387,8 +390,8 @@ async def test_real_download_status_failure_is_skipped_without_secret_logs(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("count", [2, 12])
|
||||
async def test_animation_reports_montage_including_cache_hits(tmp_path, count):
|
||||
"""Animated inputs report a montage even when its bytes are cached."""
|
||||
async def test_repeated_animation_preparation_reports_montage(tmp_path, count):
|
||||
"""Repeated animated inputs produce independent montage previews."""
|
||||
frames = [Image.new("RGB", (12, 8), (index * 20, 0, 0)) for index in range(count)]
|
||||
source = tmp_path / "animation.gif"
|
||||
frames[0].save(
|
||||
@@ -401,11 +404,11 @@ async def test_animation_reports_montage_including_cache_hits(tmp_path, count):
|
||||
assert prepared and prepared[1] is True
|
||||
assert Path(prepared[0]).is_file()
|
||||
|
||||
cached = await media.prepare_model_image(
|
||||
repeated = await media.prepare_model_image(
|
||||
str(source), max_size=1280, output_dir=tmp_path
|
||||
)
|
||||
assert cached and cached[1] is True
|
||||
assert Path(cached[0]).read_bytes() == Path(prepared[0]).read_bytes()
|
||||
assert repeated and repeated[1] is True
|
||||
assert Path(repeated[0]).read_bytes() == Path(prepared[0]).read_bytes()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -5,6 +5,7 @@ import base64
|
||||
import copy
|
||||
import inspect
|
||||
import os
|
||||
import random
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -195,7 +196,7 @@ async def process_event(harness, event, *, preprocess_first=False, stage=None):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enabled", [None, True, False])
|
||||
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "BMP", "WEBP", "GIF"])
|
||||
async def test_profile_toggle_and_preprocess_to_first_model(
|
||||
async def test_legacy_toggle_does_not_disable_preparation(
|
||||
harness, tmp_path, monkeypatch, enabled, fmt
|
||||
):
|
||||
source = source_image(tmp_path, fmt)
|
||||
@@ -205,23 +206,13 @@ async def test_profile_toggle_and_preprocess_to_first_model(
|
||||
harness.config["provider_settings"].pop("image_compress_enabled", None)
|
||||
else:
|
||||
harness.config["provider_settings"]["image_compress_enabled"] = enabled
|
||||
if enabled is False:
|
||||
monkeypatch.setattr(
|
||||
media,
|
||||
"_read_valid_cached_image_bytes",
|
||||
lambda *args: pytest.fail("disabled must not query derived cache"),
|
||||
)
|
||||
await process_event(harness, event, preprocess_first=True)
|
||||
assert len(harness.captured) == 1
|
||||
req = harness.captured[0].req
|
||||
with PILImage.open(req.image_urls[0]) as image:
|
||||
expected = (
|
||||
fmt if enabled is False else (fmt if fmt in {"JPEG", "PNG"} else "JPEG")
|
||||
)
|
||||
expected = fmt if fmt in {"JPEG", "PNG"} else "JPEG"
|
||||
assert image.format == expected
|
||||
assert image.size == (
|
||||
(90, 45) if enabled is not False and fmt == "GIF" else (60, 30)
|
||||
)
|
||||
assert image.size == ((90, 45) if fmt == "GIF" else (60, 30))
|
||||
if fmt == "PNG":
|
||||
assert image.getpixel((0, 0))[3] == 128
|
||||
assert source.read_bytes() == original
|
||||
@@ -246,10 +237,10 @@ async def test_profile_toggle_and_preprocess_to_first_model(
|
||||
("local", "cua", "PNG"),
|
||||
],
|
||||
)
|
||||
async def test_cua_runtime_keeps_input_image_geometry(
|
||||
async def test_all_runtimes_apply_configured_image_limit(
|
||||
harness, tmp_path, runtime, booter, fmt
|
||||
):
|
||||
"""CUA pixel tools read coordinates 1:1, so only the resize is lifted."""
|
||||
"""User attachments obey the image limit for every computer runtime."""
|
||||
path = tmp_path / f"big.{fmt.lower()}"
|
||||
PILImage.new("RGB", (200, 100), "red").save(path, fmt)
|
||||
original = path.read_bytes()
|
||||
@@ -259,16 +250,10 @@ async def test_cua_runtime_keeps_input_image_geometry(
|
||||
await process_event(harness, event, preprocess_first=True)
|
||||
assert len(harness.captured) == 1
|
||||
req = harness.captured[0].req
|
||||
gated = runtime == "sandbox" and booter == "cua"
|
||||
with PILImage.open(req.image_urls[0]) as image:
|
||||
assert image.size == ((200, 100) if gated else (90, 45))
|
||||
if gated:
|
||||
# A compliant source is reused byte-exact; other formats are still
|
||||
# re-encoded (to JPEG) without any resize.
|
||||
expected = fmt if fmt in {"JPEG", "PNG"} else "JPEG"
|
||||
assert image.format == expected
|
||||
if gated and fmt == "PNG":
|
||||
assert Path(req.image_urls[0]).read_bytes() == original
|
||||
assert image.size == (90, 45)
|
||||
assert image.format == "JPEG"
|
||||
assert path.read_bytes() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -280,10 +265,10 @@ async def test_cua_runtime_keeps_input_image_geometry(
|
||||
("sandbox", "shipyard_neo", True),
|
||||
],
|
||||
)
|
||||
async def test_cua_oversize_image_warns(
|
||||
async def test_cua_inputs_obey_byte_limit(
|
||||
harness, tmp_path, monkeypatch, runtime, booter, big
|
||||
):
|
||||
"""Byte-exact CUA passthrough warns when an image may exceed upload limits."""
|
||||
"""Large user inputs are bounded even in CUA sessions."""
|
||||
dims = (1500, 1500) if big else (60, 30)
|
||||
path = tmp_path / "shot.png"
|
||||
if big:
|
||||
@@ -302,11 +287,11 @@ async def test_cua_oversize_image_warns(
|
||||
await process_event(harness, event, preprocess_first=True)
|
||||
assert len(harness.captured) == 1
|
||||
req = harness.captured[0].req
|
||||
gated = runtime == "sandbox" and booter == "cua"
|
||||
if gated:
|
||||
assert Path(req.image_urls[0]).read_bytes() == original
|
||||
warned = any("upload limits" in str(call) for call in warning.call_args_list)
|
||||
assert warned == (big and gated)
|
||||
assert Path(req.image_urls[0]).stat().st_size < 512 * 1024
|
||||
with PILImage.open(req.image_urls[0]) as image:
|
||||
assert max(image.size) <= 90
|
||||
assert path.read_bytes() == original
|
||||
assert not any("upload limits" in str(call) for call in warning.call_args_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -336,12 +321,12 @@ async def test_animation_montage_notice_reaches_model(harness, tmp_path):
|
||||
|
||||
still = tmp_path / "still.png"
|
||||
PILImage.new("RGB", (60, 30), "red").save(still)
|
||||
# The second animated request reuses the montage cache.
|
||||
# Legacy settings cannot disable animation preparation.
|
||||
for source, enabled, expected in (
|
||||
(animated, True, True),
|
||||
(animated, True, True),
|
||||
(still, True, False),
|
||||
(animated, False, False),
|
||||
(animated, False, True),
|
||||
):
|
||||
harness.config["provider_settings"]["image_compress_enabled"] = enabled
|
||||
await process_event(harness, make_event([Image(file=str(source))]))
|
||||
@@ -369,7 +354,7 @@ async def test_profile_reload_and_concurrent_requests(harness, tmp_path):
|
||||
harness, make_event([Image(file=str(source))], session="first")
|
||||
)
|
||||
other_config = copy.deepcopy(harness.config)
|
||||
other_config["provider_settings"]["image_compress_enabled"] = False
|
||||
other_config["provider_settings"]["image_compress_options"]["max_size"] = 60
|
||||
other_ctx = SimpleNamespace(
|
||||
astrbot_config=other_config, plugin_manager=harness.ctx.plugin_manager
|
||||
)
|
||||
@@ -383,7 +368,7 @@ async def test_profile_reload_and_concurrent_requests(harness, tmp_path):
|
||||
),
|
||||
process_event(
|
||||
harness,
|
||||
make_event([Image(file=str(source))], session="off"),
|
||||
make_event([Image(file=str(source))], session="other"),
|
||||
stage=other_stage,
|
||||
),
|
||||
)
|
||||
@@ -392,7 +377,8 @@ async def test_profile_reload_and_concurrent_requests(harness, tmp_path):
|
||||
}
|
||||
with PILImage.open(results["small"].image_urls[0]) as image:
|
||||
assert max(image.size) == 30
|
||||
assert Path(results["off"].image_urls[0]).read_bytes() == source.read_bytes()
|
||||
with PILImage.open(results["other"].image_urls[0]) as image:
|
||||
assert max(image.size) == 60
|
||||
assert harness.provider.provider_config == before
|
||||
|
||||
|
||||
@@ -479,9 +465,8 @@ async def test_plugin_request_extra_metadata_hook_and_history(
|
||||
assert all(p["image_url"].get("id") != "dict-id" for p in images)
|
||||
paths = list(event._temporary_local_files)
|
||||
event.cleanup_temporary_local_files()
|
||||
cache_files = list((harness.work / media.CONVERT_CACHE_DIR_NAME).glob("*.img"))
|
||||
assert all(not Path(path).exists() for path in paths)
|
||||
assert cache_files and all(path.exists() for path in cache_files)
|
||||
assert not list(harness.work.rglob("model_image_*"))
|
||||
assert all(
|
||||
base64.b64decode(p["image_url"]["url"].split(",", 1)[1]).startswith(b"\xff\xd8")
|
||||
for p in images
|
||||
@@ -495,6 +480,98 @@ async def test_plugin_request_extra_metadata_hook_and_history(
|
||||
assert all(part["source"]["media_type"] == "image/jpeg" for part in visual)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("entry", ["attachment", "quote", "plugin", "hook"])
|
||||
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "GIF"])
|
||||
async def test_provider_and_history_keep_bounded_previews_and_original_paths(
|
||||
harness, tmp_path, monkeypatch, entry, fmt
|
||||
):
|
||||
source = tmp_path / f"original.{fmt.lower()}"
|
||||
pixels = random.Random(9703).randbytes(1280 * 960 * 3)
|
||||
image = PILImage.frombytes("RGB", (1280, 960), pixels)
|
||||
if fmt == "PNG":
|
||||
image.putalpha(128)
|
||||
if fmt == "GIF":
|
||||
image.save(
|
||||
source,
|
||||
fmt,
|
||||
save_all=True,
|
||||
append_images=[image.transpose(PILImage.Transpose.FLIP_LEFT_RIGHT)],
|
||||
)
|
||||
else:
|
||||
image.save(source, fmt, quality=100)
|
||||
original = source.read_bytes()
|
||||
assert len(original) >= 512 * 1024
|
||||
harness.config["provider_settings"]["image_compress_options"] = {
|
||||
"max_size": 1280,
|
||||
"quality": 100,
|
||||
}
|
||||
harness.config["provider_settings"]["image_compress_enabled"] = False
|
||||
part = Image(file=str(source))
|
||||
event = make_event(
|
||||
[Reply(id="quoted", chain=[part])] if entry == "quote" else [part]
|
||||
)
|
||||
if entry in {"plugin", "hook"}:
|
||||
event = make_event()
|
||||
original_request = ProviderRequest(
|
||||
prompt="describe",
|
||||
image_urls=[str(source)],
|
||||
conversation=harness.context.conversation_manager.get_conversation.return_value,
|
||||
extra_user_content_parts=[
|
||||
TextPart(text=f"[Image Attachment: path {source}]")
|
||||
],
|
||||
)
|
||||
if entry == "plugin":
|
||||
event.set_extra("provider_request", original_request)
|
||||
else:
|
||||
|
||||
async def hook(event, kind, *args):
|
||||
if kind == EventType.OnLLMRequestEvent:
|
||||
request = args[0]
|
||||
request.image_urls = original_request.image_urls
|
||||
request.extra_user_content_parts = (
|
||||
original_request.extra_user_content_parts
|
||||
)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(internal, "call_event_hook", hook)
|
||||
await process_event(harness, event, preprocess_first=True)
|
||||
req = harness.captured[0].req
|
||||
preview = Path(req.image_urls[0])
|
||||
assert preview != source and preview.stat().st_size < 512 * 1024
|
||||
texts = [
|
||||
part.text for part in req.extra_user_content_parts if isinstance(part, TextPart)
|
||||
]
|
||||
assert any(str(source) in text for text in texts)
|
||||
assert all(str(preview) not in text for text in texts)
|
||||
provider_messages = harness.provider.text_chat.await_args.kwargs["contexts"]
|
||||
history = (
|
||||
harness.context.conversation_manager.update_conversation.await_args.kwargs[
|
||||
"history"
|
||||
]
|
||||
)
|
||||
for messages in (provider_messages, history):
|
||||
images = [
|
||||
part
|
||||
for message in messages
|
||||
if isinstance(message.get("content"), list)
|
||||
for part in message["content"]
|
||||
if part["type"] == "image_url"
|
||||
]
|
||||
assert images
|
||||
for part in images:
|
||||
data_uri = part["image_url"]["url"]
|
||||
data = base64.b64decode(data_uri.split(",", 1)[1])
|
||||
assert len(data) < 512 * 1024
|
||||
assert data == preview.read_bytes()
|
||||
assert data_uri.startswith(
|
||||
"data:image/png;" if fmt == "PNG" else "data:image/jpeg;"
|
||||
)
|
||||
event.cleanup_temporary_local_files()
|
||||
assert not preview.exists()
|
||||
assert source.read_bytes() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("plugin", [False, True])
|
||||
async def test_caption_first_call_uses_prepared_quote(harness, tmp_path, plugin):
|
||||
@@ -660,15 +737,11 @@ async def test_localized_reference_lifetime_and_ownership(
|
||||
path = Path(harness.captured[0].req.image_urls[0])
|
||||
assert path.is_file()
|
||||
assert str(source) not in event._temporary_local_files
|
||||
assert (str(path) in event._temporary_local_files) == (
|
||||
enabled or reference != "file"
|
||||
)
|
||||
assert str(path) in event._temporary_local_files
|
||||
with PILImage.open(path) as image:
|
||||
assert image.format == ("JPEG" if enabled else "GIF")
|
||||
if not enabled:
|
||||
assert path.read_bytes() == source.read_bytes()
|
||||
assert image.format == "JPEG"
|
||||
event.cleanup_temporary_local_files()
|
||||
assert path.exists() == (reference == "file" and not enabled)
|
||||
assert not path.exists()
|
||||
assert source.is_file()
|
||||
|
||||
|
||||
@@ -819,18 +892,18 @@ async def test_attachment_sources_survive_event_cleanup(
|
||||
visual_path = Path(req.image_urls[0])
|
||||
assert attachment_path.read_bytes() == original
|
||||
assert str(attachment_path) not in event._temporary_local_files
|
||||
assert (visual_path != attachment_path) == enabled
|
||||
assert visual_path != attachment_path
|
||||
with PILImage.open(visual_path) as visual_image:
|
||||
assert visual_image.format == ("JPEG" if enabled else fmt)
|
||||
assert max(visual_image.size) == (12 if enabled else 60)
|
||||
assert visual_image.format == "JPEG"
|
||||
assert max(visual_image.size) == 12
|
||||
|
||||
owned = [Path(path) for path in event._temporary_local_files]
|
||||
assert len(owned) == 1 + int(enabled)
|
||||
assert len(owned) == 2
|
||||
assert unrelated in owned
|
||||
event.cleanup_temporary_local_files()
|
||||
assert attachment_path.read_bytes() == original
|
||||
assert all(not path.exists() for path in owned)
|
||||
assert visual_path.exists() == (not enabled)
|
||||
assert not visual_path.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user