/** * Media Handler Module * Story 1.3: Content Migration Script * * Downloads images from URLs and uploads to Payload CMS */ import type { Payload, File } from 'payload' import type { MediaDownloadResult } from './types' import { getFilenameFromUrl, getFileExtension } from './utils' // ============================================================ // DOWNLOAD MEDIA // ============================================================ /** * Download an image from URL */ export async function downloadImage( url: string, retries: number = 3, ): Promise { let lastError: Error | undefined for (let i = 0; i < retries; i++) { try { const response = await fetch(url, { method: 'GET', headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', }, signal: AbortSignal.timeout(30000), // 30 second timeout }) if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`) } const buffer = Buffer.from(await response.arrayBuffer()) const filename = getFilenameFromUrl(url) return { success: true, url, buffer, filename, } } catch (error) { lastError = error as Error // Wait before retry (exponential backoff) if (i < retries - 1) { await new Promise((resolve) => setTimeout(resolve, Math.pow(2, i) * 1000)) } } } return { success: false, url, error: lastError?.message || 'Unknown error', } } /** * Download multiple images in parallel batches */ export async function downloadImages( urls: string[], batchSize: number = 5, ): Promise { const results: MediaDownloadResult[] = [] for (let i = 0; i < urls.length; i += batchSize) { const batch = urls.slice(i, i + batchSize) const batchResults = await Promise.all(batch.map((url) => downloadImage(url))) results.push(...batchResults) } return results } // ============================================================ // UPLOAD TO PAYLOAD CMS // ============================================================ /** * Upload a single image to Payload CMS Media collection */ export async function uploadToMedia( payload: Payload, downloadResult: MediaDownloadResult, ): Promise<{ success: boolean; id?: string; error?: string }> { if (!downloadResult.success || !downloadResult.buffer || !downloadResult.filename) { return { success: false, error: downloadResult.error || 'Invalid download result', } } try { const file: File = { name: downloadResult.filename, data: downloadResult.buffer, mimetype: `image/${getFileExtension(downloadResult.filename)}`, size: downloadResult.buffer.length, } const result = await payload.create({ collection: 'media', file, data: { alt: downloadResult.filename, }, }) return { success: true, id: result.id, } } catch (error) { return { success: false, error: error instanceof Error ? error.message : 'Unknown error', } } } /** * Check if media already exists by filename */ export async function findMediaByFilename( payload: Payload, filename: string, ): Promise { try { const result = await payload.find({ collection: 'media', where: { filename: { equals: filename }, }, limit: 1, depth: 0, }) if (result.docs && result.docs.length > 0) { return result.docs[0].id } return null } catch { return null } } // ============================================================ // BATCH PROCESSING // ============================================================ /** * Process all media URLs: download and upload to Payload CMS * Returns a map of original URL to Media ID */ export async function processMediaUrls( payload: Payload, urls: string[], options: { batchSize?: number retries?: number onProgress?: (current: number, total: number) => void } = {}, ): Promise> { const { batchSize = 5, onProgress } = options const urlToIdMap = new Map() // Filter out empty URLs const validUrls = urls.filter((url) => url && isValidImageUrl(url)) for (let i = 0; i < validUrls.length; i += batchSize) { const batch = validUrls.slice(i, i + batchSize) // Download batch const downloadResults = await Promise.all( batch.map((url) => downloadImage(url, options.retries || 3)), ) // Upload each downloaded image for (const result of downloadResults) { if (result.success && result.buffer && result.filename) { // Check if already exists const existingId = await findMediaByFilename(payload, result.filename) if (existingId) { urlToIdMap.set(result.url, existingId) } else { // Upload new const uploadResult = await uploadToMedia(payload, result) if (uploadResult.success && uploadResult.id) { urlToIdMap.set(result.url, uploadResult.id) } } } // Report progress if (onProgress) { onProgress(urlToIdMap.size, validUrls.length) } } } return urlToIdMap } /** * Process a single media URL with caching */ const mediaCache = new Map() export async function getOrCreateMedia( payload: Payload, url: string, ): Promise { // Check cache first if (mediaCache.has(url)) { return mediaCache.get(url) } if (!url || !isValidImageUrl(url)) { return undefined } const downloadResult = await downloadImage(url) if (!downloadResult.success || !downloadResult.buffer || !downloadResult.filename) { return undefined } // Check if already exists const existingId = await findMediaByFilename(payload, downloadResult.filename) if (existingId) { mediaCache.set(url, existingId) return existingId } // Upload new const uploadResult = await uploadToMedia(payload, downloadResult) if (uploadResult.success && uploadResult.id) { mediaCache.set(url, uploadResult.id) return uploadResult.id } return undefined } // ============================================================ // UTILITIES // ============================================================ /** * Check if URL is a valid image */ function isValidImageUrl(url: string): boolean { if (!url) { return false } try { const urlObj = new URL(url) const pathname = urlObj.pathname.toLowerCase() return ( pathname.endsWith('.jpg') || pathname.endsWith('.jpeg') || pathname.endsWith('.png') || pathname.endsWith('.gif') || pathname.endsWith('.webp') || pathname.endsWith('.svg') ) } catch { return false } } /** * Clear media cache (useful for testing) */ export function clearMediaCache(): void { mediaCache.clear() }