fix: Add Base64 image format conversion support

Extends conversion logic to handle Base64-encoded images (e.g., from clipboard).
Previously, unsupported formats like HEIC/RAW/PSD in Base64 form bypassed
the conversion check and caused failures at multimodal-looker agent.

Changes:
- Add convertBase64ImageToJpeg() function in image-converter.ts
- Save Base64 data to temp file, convert, read back as Base64
- Update tools.ts to check and convert Base64 images when needed
- Ensure proper cleanup of all temporary files

Testing:
- All tests pass (29/29)
- Verified with 1.7MB HEIC file converted from Base64
- Type checking passes
This commit is contained in:
XIN PENG
2026-02-16 11:08:25 -08:00
parent ae19ff60cf
commit 116ca090e0
2 changed files with 65 additions and 9 deletions

View File

@@ -1,5 +1,5 @@
import { execSync } from "node:child_process"
import { existsSync, mkdtempSync, unlinkSync } from "node:fs"
import { existsSync, mkdtempSync, unlinkSync, writeFileSync, readFileSync } from "node:fs"
import { tmpdir } from "node:os"
import { join } from "node:path"
import { log } from "../../shared"
@@ -112,3 +112,38 @@ export function cleanupConvertedImage(filePath: string): void {
log(`[image-converter] Failed to cleanup ${filePath}: ${error}`)
}
}
export function convertBase64ImageToJpeg(
base64Data: string,
mimeType: string
): { base64: string; tempFiles: string[] } {
const tempDir = mkdtempSync(join(tmpdir(), "opencode-b64-"))
const inputExt = mimeType.split("/")[1] || "bin"
const inputPath = join(tempDir, `input.${inputExt}`)
const tempFiles: string[] = [inputPath]
try {
const cleanBase64 = base64Data.replace(/^data:[^;]+;base64,/, "")
const buffer = Buffer.from(cleanBase64, "base64")
writeFileSync(inputPath, buffer)
log(`[image-converter] Converting Base64 ${mimeType} to JPEG`)
const outputPath = convertImageToJpeg(inputPath, mimeType)
tempFiles.push(outputPath)
const convertedBuffer = readFileSync(outputPath)
const convertedBase64 = convertedBuffer.toString("base64")
log(`[image-converter] Base64 conversion successful`)
return { base64: convertedBase64, tempFiles }
} catch (error) {
tempFiles.forEach(file => {
try {
if (existsSync(file)) unlinkSync(file)
} catch {}
})
throw error
}
}

View File

@@ -16,6 +16,7 @@ import { resolveMultimodalLookerAgentMetadata } from "./multimodal-agent-metadat
import {
needsConversion,
convertImageToJpeg,
convertBase64ImageToJpeg,
cleanupConvertedImage,
} from "./image-converter"
@@ -47,17 +48,36 @@ export function createLookAt(ctx: PluginInput): ToolDefinition {
let mimeType: string
let filePart: { type: "file"; mime: string; url: string; filename: string }
let tempFilePath: string | null = null
let tempFilesToCleanup: string[] = []
try {
if (imageData) {
mimeType = inferMimeTypeFromBase64(imageData)
filePart = {
type: "file",
mime: mimeType,
url: `data:${mimeType};base64,${extractBase64Data(imageData)}`,
filename: `clipboard-image.${mimeType.split("/")[1] || "png"}`,
}
} else if (filePath) {
mimeType = inferMimeTypeFromBase64(imageData)
let finalBase64Data = extractBase64Data(imageData)
let finalMimeType = mimeType
if (needsConversion(mimeType)) {
log(`[look_at] Detected unsupported Base64 format: ${mimeType}, converting to JPEG...`)
try {
const { base64, tempFiles } = convertBase64ImageToJpeg(imageData, mimeType)
finalBase64Data = base64
finalMimeType = "image/jpeg"
tempFilesToCleanup = tempFiles
log(`[look_at] Base64 conversion successful`)
} catch (conversionError) {
log(`[look_at] Base64 conversion failed: ${conversionError}`)
return `Error: Failed to convert Base64 image format. ${conversionError}`
}
}
filePart = {
type: "file",
mime: finalMimeType,
url: `data:${finalMimeType};base64,${finalBase64Data}`,
filename: `clipboard-image.${finalMimeType.split("/")[1] || "png"}`,
}
} else if (filePath) {
mimeType = inferMimeTypeFromFilePath(filePath)
let actualFilePath = filePath
@@ -177,6 +197,7 @@ Original error: ${createResult.error}`
if (tempFilePath) {
cleanupConvertedImage(tempFilePath)
}
tempFilesToCleanup.forEach(file => cleanupConvertedImage(file))
}
},
})