fix: Add Base64 image format conversion support
Extends conversion logic to handle Base64-encoded images (e.g., from clipboard). Previously, unsupported formats like HEIC/RAW/PSD in Base64 form bypassed the conversion check and caused failures at multimodal-looker agent. Changes: - Add convertBase64ImageToJpeg() function in image-converter.ts - Save Base64 data to temp file, convert, read back as Base64 - Update tools.ts to check and convert Base64 images when needed - Ensure proper cleanup of all temporary files Testing: - All tests pass (29/29) - Verified with 1.7MB HEIC file converted from Base64 - Type checking passes
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { execSync } from "node:child_process"
|
||||
import { existsSync, mkdtempSync, unlinkSync } from "node:fs"
|
||||
import { existsSync, mkdtempSync, unlinkSync, writeFileSync, readFileSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { log } from "../../shared"
|
||||
@@ -112,3 +112,38 @@ export function cleanupConvertedImage(filePath: string): void {
|
||||
log(`[image-converter] Failed to cleanup ${filePath}: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
export function convertBase64ImageToJpeg(
|
||||
base64Data: string,
|
||||
mimeType: string
|
||||
): { base64: string; tempFiles: string[] } {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "opencode-b64-"))
|
||||
const inputExt = mimeType.split("/")[1] || "bin"
|
||||
const inputPath = join(tempDir, `input.${inputExt}`)
|
||||
const tempFiles: string[] = [inputPath]
|
||||
|
||||
try {
|
||||
const cleanBase64 = base64Data.replace(/^data:[^;]+;base64,/, "")
|
||||
const buffer = Buffer.from(cleanBase64, "base64")
|
||||
writeFileSync(inputPath, buffer)
|
||||
|
||||
log(`[image-converter] Converting Base64 ${mimeType} to JPEG`)
|
||||
|
||||
const outputPath = convertImageToJpeg(inputPath, mimeType)
|
||||
tempFiles.push(outputPath)
|
||||
|
||||
const convertedBuffer = readFileSync(outputPath)
|
||||
const convertedBase64 = convertedBuffer.toString("base64")
|
||||
|
||||
log(`[image-converter] Base64 conversion successful`)
|
||||
|
||||
return { base64: convertedBase64, tempFiles }
|
||||
} catch (error) {
|
||||
tempFiles.forEach(file => {
|
||||
try {
|
||||
if (existsSync(file)) unlinkSync(file)
|
||||
} catch {}
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import { resolveMultimodalLookerAgentMetadata } from "./multimodal-agent-metadat
|
||||
import {
|
||||
needsConversion,
|
||||
convertImageToJpeg,
|
||||
convertBase64ImageToJpeg,
|
||||
cleanupConvertedImage,
|
||||
} from "./image-converter"
|
||||
|
||||
@@ -47,17 +48,36 @@ export function createLookAt(ctx: PluginInput): ToolDefinition {
|
||||
let mimeType: string
|
||||
let filePart: { type: "file"; mime: string; url: string; filename: string }
|
||||
let tempFilePath: string | null = null
|
||||
let tempFilesToCleanup: string[] = []
|
||||
|
||||
try {
|
||||
if (imageData) {
|
||||
mimeType = inferMimeTypeFromBase64(imageData)
|
||||
filePart = {
|
||||
type: "file",
|
||||
mime: mimeType,
|
||||
url: `data:${mimeType};base64,${extractBase64Data(imageData)}`,
|
||||
filename: `clipboard-image.${mimeType.split("/")[1] || "png"}`,
|
||||
}
|
||||
} else if (filePath) {
|
||||
mimeType = inferMimeTypeFromBase64(imageData)
|
||||
|
||||
let finalBase64Data = extractBase64Data(imageData)
|
||||
let finalMimeType = mimeType
|
||||
|
||||
if (needsConversion(mimeType)) {
|
||||
log(`[look_at] Detected unsupported Base64 format: ${mimeType}, converting to JPEG...`)
|
||||
try {
|
||||
const { base64, tempFiles } = convertBase64ImageToJpeg(imageData, mimeType)
|
||||
finalBase64Data = base64
|
||||
finalMimeType = "image/jpeg"
|
||||
tempFilesToCleanup = tempFiles
|
||||
log(`[look_at] Base64 conversion successful`)
|
||||
} catch (conversionError) {
|
||||
log(`[look_at] Base64 conversion failed: ${conversionError}`)
|
||||
return `Error: Failed to convert Base64 image format. ${conversionError}`
|
||||
}
|
||||
}
|
||||
|
||||
filePart = {
|
||||
type: "file",
|
||||
mime: finalMimeType,
|
||||
url: `data:${finalMimeType};base64,${finalBase64Data}`,
|
||||
filename: `clipboard-image.${finalMimeType.split("/")[1] || "png"}`,
|
||||
}
|
||||
} else if (filePath) {
|
||||
mimeType = inferMimeTypeFromFilePath(filePath)
|
||||
|
||||
let actualFilePath = filePath
|
||||
@@ -177,6 +197,7 @@ Original error: ${createResult.error}`
|
||||
if (tempFilePath) {
|
||||
cleanupConvertedImage(tempFilePath)
|
||||
}
|
||||
tempFilesToCleanup.forEach(file => cleanupConvertedImage(file))
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user