feat(backend): update collections, config and migration tools
Update Payload CMS configuration, collections (Audit, Posts), and add migration scripts/reports.
This commit is contained in:
286
apps/backend/scripts/migration/mediaHandler.ts
Normal file
286
apps/backend/scripts/migration/mediaHandler.ts
Normal file
@@ -0,0 +1,286 @@
|
||||
/**
|
||||
* Media Handler Module
|
||||
* Story 1.3: Content Migration Script
|
||||
*
|
||||
* Downloads images from URLs and uploads to Payload CMS
|
||||
*/
|
||||
|
||||
import type { Payload, File } from 'payload'
|
||||
import type { MediaDownloadResult } from './types'
|
||||
import { getFilenameFromUrl, getFileExtension } from './utils'
|
||||
|
||||
// ============================================================
|
||||
// DOWNLOAD MEDIA
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Download an image from URL
|
||||
*/
|
||||
export async function downloadImage(
|
||||
url: string,
|
||||
retries: number = 3,
|
||||
): Promise<MediaDownloadResult> {
|
||||
let lastError: Error | undefined
|
||||
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
},
|
||||
signal: AbortSignal.timeout(30000), // 30 second timeout
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer())
|
||||
const filename = getFilenameFromUrl(url)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
url,
|
||||
buffer,
|
||||
filename,
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error as Error
|
||||
// Wait before retry (exponential backoff)
|
||||
if (i < retries - 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.pow(2, i) * 1000))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
url,
|
||||
error: lastError?.message || 'Unknown error',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download multiple images in parallel batches
|
||||
*/
|
||||
export async function downloadImages(
|
||||
urls: string[],
|
||||
batchSize: number = 5,
|
||||
): Promise<MediaDownloadResult[]> {
|
||||
const results: MediaDownloadResult[] = []
|
||||
|
||||
for (let i = 0; i < urls.length; i += batchSize) {
|
||||
const batch = urls.slice(i, i + batchSize)
|
||||
const batchResults = await Promise.all(batch.map((url) => downloadImage(url)))
|
||||
results.push(...batchResults)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// UPLOAD TO PAYLOAD CMS
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Upload a single image to Payload CMS Media collection
|
||||
*/
|
||||
export async function uploadToMedia(
|
||||
payload: Payload,
|
||||
downloadResult: MediaDownloadResult,
|
||||
): Promise<{ success: boolean; id?: string; error?: string }> {
|
||||
if (!downloadResult.success || !downloadResult.buffer || !downloadResult.filename) {
|
||||
return {
|
||||
success: false,
|
||||
error: downloadResult.error || 'Invalid download result',
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const file: File = {
|
||||
name: downloadResult.filename,
|
||||
data: downloadResult.buffer,
|
||||
mimetype: `image/${getFileExtension(downloadResult.filename)}`,
|
||||
size: downloadResult.buffer.length,
|
||||
}
|
||||
|
||||
const result = await payload.create({
|
||||
collection: 'media',
|
||||
file,
|
||||
data: {
|
||||
alt: downloadResult.filename,
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
success: true,
|
||||
id: result.id,
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if media already exists by filename
|
||||
*/
|
||||
export async function findMediaByFilename(
|
||||
payload: Payload,
|
||||
filename: string,
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const result = await payload.find({
|
||||
collection: 'media',
|
||||
where: {
|
||||
filename: { equals: filename },
|
||||
},
|
||||
limit: 1,
|
||||
depth: 0,
|
||||
})
|
||||
|
||||
if (result.docs && result.docs.length > 0) {
|
||||
return result.docs[0].id
|
||||
}
|
||||
|
||||
return null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// BATCH PROCESSING
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Process all media URLs: download and upload to Payload CMS
|
||||
* Returns a map of original URL to Media ID
|
||||
*/
|
||||
export async function processMediaUrls(
|
||||
payload: Payload,
|
||||
urls: string[],
|
||||
options: {
|
||||
batchSize?: number
|
||||
retries?: number
|
||||
onProgress?: (current: number, total: number) => void
|
||||
} = {},
|
||||
): Promise<Map<string, string>> {
|
||||
const { batchSize = 5, onProgress } = options
|
||||
const urlToIdMap = new Map<string, string>()
|
||||
|
||||
// Filter out empty URLs
|
||||
const validUrls = urls.filter((url) => url && isValidImageUrl(url))
|
||||
|
||||
for (let i = 0; i < validUrls.length; i += batchSize) {
|
||||
const batch = validUrls.slice(i, i + batchSize)
|
||||
|
||||
// Download batch
|
||||
const downloadResults = await Promise.all(
|
||||
batch.map((url) => downloadImage(url, options.retries || 3)),
|
||||
)
|
||||
|
||||
// Upload each downloaded image
|
||||
for (const result of downloadResults) {
|
||||
if (result.success && result.buffer && result.filename) {
|
||||
// Check if already exists
|
||||
const existingId = await findMediaByFilename(payload, result.filename)
|
||||
|
||||
if (existingId) {
|
||||
urlToIdMap.set(result.url, existingId)
|
||||
} else {
|
||||
// Upload new
|
||||
const uploadResult = await uploadToMedia(payload, result)
|
||||
if (uploadResult.success && uploadResult.id) {
|
||||
urlToIdMap.set(result.url, uploadResult.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress
|
||||
if (onProgress) {
|
||||
onProgress(urlToIdMap.size, validUrls.length)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return urlToIdMap
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single media URL with caching
|
||||
*/
|
||||
const mediaCache = new Map<string, string>()
|
||||
|
||||
export async function getOrCreateMedia(
|
||||
payload: Payload,
|
||||
url: string,
|
||||
): Promise<string | undefined> {
|
||||
// Check cache first
|
||||
if (mediaCache.has(url)) {
|
||||
return mediaCache.get(url)
|
||||
}
|
||||
|
||||
if (!url || !isValidImageUrl(url)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const downloadResult = await downloadImage(url)
|
||||
if (!downloadResult.success || !downloadResult.buffer || !downloadResult.filename) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Check if already exists
|
||||
const existingId = await findMediaByFilename(payload, downloadResult.filename)
|
||||
if (existingId) {
|
||||
mediaCache.set(url, existingId)
|
||||
return existingId
|
||||
}
|
||||
|
||||
// Upload new
|
||||
const uploadResult = await uploadToMedia(payload, downloadResult)
|
||||
if (uploadResult.success && uploadResult.id) {
|
||||
mediaCache.set(url, uploadResult.id)
|
||||
return uploadResult.id
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// UTILITIES
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Check if URL is a valid image
|
||||
*/
|
||||
function isValidImageUrl(url: string): boolean {
|
||||
if (!url) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
const urlObj = new URL(url)
|
||||
const pathname = urlObj.pathname.toLowerCase()
|
||||
return (
|
||||
pathname.endsWith('.jpg') ||
|
||||
pathname.endsWith('.jpeg') ||
|
||||
pathname.endsWith('.png') ||
|
||||
pathname.endsWith('.gif') ||
|
||||
pathname.endsWith('.webp') ||
|
||||
pathname.endsWith('.svg')
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear media cache (useful for testing)
|
||||
*/
|
||||
export function clearMediaCache(): void {
|
||||
mediaCache.clear()
|
||||
}
|
||||
Reference in New Issue
Block a user