Refactor SVG processing scripts and add new functionality

- Updated `svg-cleanup.js` to improve SVG file processing and validation.
- Refactored `update-data.js` to streamline the data update process and integrate new scripts for generating image variants and syncing data files.
- Introduced `generate-variants.js` to handle the conversion of SVG files to PNG and JPG formats.
- Created `sync-data.js` to synchronize logo data with the filesystem, ensuring accurate representation of available images.
- Enhanced error handling and logging throughout the scripts for better debugging and user feedback.
- Added support for processing all collections or a specific collection based on command-line arguments or environment variables.
This commit is contained in:
sHa
2025-06-19 17:09:45 +03:00
parent 1db0f1cbe9
commit 374ece5142
11 changed files with 424 additions and 386 deletions

View File

@@ -0,0 +1,131 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { Resvg } = require('@resvg/resvg-js');
// Use collections from src/collections.js
const { collections } = require('../src/collections.js');
// Accept collection as a CLI arg or env var
const collectionArg = process.argv.find(arg => arg.startsWith('--collection='));
const collectionName = collectionArg ? collectionArg.split('=')[1] : (process.env.COLLECTION || 'logos');
// Get file name without extension
function getBaseName(filename) {
return path.basename(filename, path.extname(filename));
}
// Clean directory (remove all contents)
function cleanDir(dir) {
if (fs.existsSync(dir)) {
for (const file of fs.readdirSync(dir)) {
if (file !== '.gitignore') {
const filePath = path.join(dir, file);
if (fs.lstatSync(filePath).isDirectory()) {
fs.rmSync(filePath, { recursive: true, force: true });
} else {
fs.unlinkSync(filePath);
}
}
}
} else {
fs.mkdirSync(dir, { recursive: true });
}
}
// Convert SVG to PNG with transparency
function svgToPng(svgBuffer, width, height) {
// No background specified to maintain transparency
const resvg = new Resvg(svgBuffer, {
fitTo: { mode: 'width', value: width || 256 }
});
const pngData = resvg.render().asPng();
return pngData;
}
// Convert SVG to JPG
function svgToJpg(svgBuffer, width, height) {
// For JPGs we need a white background since JPG doesn't support transparency
const resvg = new Resvg(svgBuffer, {
background: 'white',
fitTo: { mode: 'width', value: width || 256 }
});
const pngData = resvg.render().asPng();
return pngData;
}
// Generate PNG and JPG variants for SVG files
function generateVariants(collectionName) {
const collection = collections.find(c => c.name === collectionName);
if (!collection) {
console.error(`Collection "${collectionName}" not found`);
return;
}
const imagesDir = path.join(__dirname, '..', 'public', collection.baseDir);
const varDir = path.join(__dirname, '..', 'public', collection.varDir);
if (!fs.existsSync(imagesDir)) {
console.error(`Directory does not exist: ${imagesDir}`);
return;
}
console.log(`Generating variants for collection: ${collection.label}`);
console.log(`Source: ${imagesDir}`);
console.log(`Target: ${varDir}`);
// Clean variants directory
cleanDir(varDir);
const files = fs.readdirSync(imagesDir);
const svgFiles = files.filter(file => /\.svg$/i.test(file));
console.log(`Found ${svgFiles.length} SVG files to process`);
let processed = 0;
let errors = 0;
for (const file of svgFiles) {
const base = getBaseName(file);
const svgPath = path.join(imagesDir, file);
const pngPath = path.join(varDir, base + '.png');
const jpgPath = path.join(varDir, base + '.jpg');
try {
const svgBuffer = fs.readFileSync(svgPath);
// Generate PNG
const pngBuffer = svgToPng(svgBuffer, 256, 256);
fs.writeFileSync(pngPath, pngBuffer);
// Generate JPG
const jpgBuffer = svgToJpg(svgBuffer);
fs.writeFileSync(jpgPath, jpgBuffer);
processed++;
console.log(`✓ Generated variants for ${file}`);
} catch (e) {
errors++;
console.error(`✗ Error generating variants for ${file}:`, e.message);
}
}
console.log(`\nCompleted: ${processed} processed, ${errors} errors`);
}
// Main function
function main() {
if (collectionName === 'all') {
// Process all collections
for (const col of collections) {
generateVariants(col.name);
}
} else {
// Process single collection
generateVariants(collectionName);
}
}
// Run the script
main();

View File

@@ -106,24 +106,24 @@ function processSvgFiles(collectionName) {
}
const imagesDir = path.join(__dirname, '..', 'public', collection.baseDir);
if (!fs.existsSync(imagesDir)) {
console.error(`Directory does not exist: ${imagesDir}`);
return;
}
console.log(`Processing SVG files in collection: ${collection.label}`);
const files = fs.readdirSync(imagesDir);
const svgFiles = files.filter(file => /\.svg$/i.test(file));
console.log(`Found ${svgFiles.length} SVG files`);
for (const file of svgFiles) {
const svgPath = path.join(imagesDir, file);
validateAndFixSvg(svgPath);
}
console.log(`Completed processing SVG files for ${collection.label}`);
}

142
scripts/sync-data.js Normal file
View File

@@ -0,0 +1,142 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
// Use collections from src/collections.js
const { collections } = require('../src/collections.js');
// Accept collection as a CLI arg or env var
const collectionArg = process.argv.find(arg => arg.startsWith('--collection='));
const collectionName = collectionArg ? collectionArg.split('=')[1] : (process.env.COLLECTION || 'logos');
// Get file extension without the dot
function getFileExtension(filename) {
return path.extname(filename).slice(1).toUpperCase();
}
// Get file name without extension
function getBaseName(filename) {
return path.basename(filename, path.extname(filename));
}
// Convert filename to readable name (replace hyphens with spaces, capitalize words)
function formatName(filename) {
return getBaseName(filename)
.split(/[-_]/)
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
.join(' ');
}
// Sync data file with filesystem
function syncDataFile(collectionName) {
const collection = collections.find(c => c.name === collectionName);
if (!collection) {
console.error(`Collection "${collectionName}" not found`);
return;
}
const imagesDir = path.join(__dirname, '..', 'public', collection.baseDir);
const outputFile = path.join(__dirname, '..', 'public', collection.dataFile);
if (!fs.existsSync(imagesDir)) {
console.error(`Directory does not exist: ${imagesDir}`);
return;
}
console.log(`Syncing data file for collection: ${collection.label}`);
console.log(`Source: ${imagesDir}`);
console.log(`Data file: ${outputFile}`);
// Load existing data
let existing = [];
if (fs.existsSync(outputFile)) {
try {
existing = JSON.parse(fs.readFileSync(outputFile, 'utf8'));
} catch (e) {
console.error('Could not parse existing data file:', e);
}
}
// Get current files
const files = fs.readdirSync(imagesDir);
const logoFiles = files.filter(file =>
/\.(svg|png|jpg|jpeg)$/i.test(file)
);
const logoFilesSet = new Set(logoFiles);
// Update existing entries
let updated = 0;
let disabled = 0;
let enabled = 0;
for (const logo of existing) {
// Fix: If logo.path contains a slash, strip to filename only
if (logo.path.includes('/')) {
logo.path = logo.path.split('/').pop();
updated++;
}
// Check if file exists
if (!logoFilesSet.has(logo.path)) {
if (!logo.disable) {
logo.disable = true;
disabled++;
}
} else if (logo.disable) {
logo.disable = false;
enabled++;
}
}
// Add new entries
const existingPathsSet = new Set(existing.map(logo => logo.path));
const newLogos = logoFiles
.filter(file => !existingPathsSet.has(file))
.map(file => {
const format = getFileExtension(file);
return {
name: formatName(file),
path: file,
format: format,
disable: false
};
})
.sort((a, b) => a.name.localeCompare(b.name));
// Merge existing and new logos
const merged = [...existing, ...newLogos];
// Save updated data
try {
const data = JSON.stringify(merged, null, 2);
fs.writeFileSync(outputFile, data);
console.log(`\nSync completed:`);
console.log(`- Total entries: ${merged.length}`);
console.log(`- New entries: ${newLogos.length}`);
console.log(`- Updated paths: ${updated}`);
console.log(`- Disabled: ${disabled}`);
console.log(`- Re-enabled: ${enabled}`);
} catch (error) {
console.error('Error writing data file:', error);
}
}
// Main function
function main() {
if (collectionName === 'all') {
// Process all collections
for (const col of collections) {
syncDataFile(col.name);
}
} else {
// Process single collection
syncDataFile(collectionName);
}
}
// Run the script
main();

View File

@@ -1,352 +1,64 @@
#!/usr/bin/env node
const fs = require('fs');
const { execSync } = require('child_process');
const path = require('path');
const { Resvg } = require('@resvg/resvg-js');
// Use collections from src/collections.js
const { collections } = require('../src/collections.js');
// Accept collection as a CLI arg or env var
const collectionArg = process.argv.find(arg => arg.startsWith('--collection='));
const collectionName = collectionArg ? collectionArg.split('=')[1] : (process.env.COLLECTION || 'logos');
const collection = collections.find(c => c.name === collectionName) || collections[0];
const imagesDir = path.join(__dirname, '..', 'public', collection.baseDir);
const outputFile = path.join(__dirname, '..', 'public', collection.dataFile);
const imagesVarDir = path.join(__dirname, '..', 'public', collection.varDir);
// Remove old PNG/JPG folders if they exist
const pngDir = path.join(__dirname, '..', 'public', collection.baseDir + '-png');
const jpgDir = path.join(__dirname, '..', 'public', collection.baseDir + '-jpg');
if (fs.existsSync(pngDir)) fs.rmSync(pngDir, { recursive: true, force: true });
if (fs.existsSync(jpgDir)) fs.rmSync(jpgDir, { recursive: true, force: true });
// Get file extension without the dot
function getFileExtension(filename) {
return path.extname(filename).slice(1).toUpperCase();
}
// Get file name without extension
function getBaseName(filename) {
return path.basename(filename, path.extname(filename));
}
// Convert filename to readable name (replace hyphens with spaces, capitalize words)
function formatName(filename) {
return getBaseName(filename)
.split(/[-_]/)
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
.join(' ');
}
// Clean directory (remove all contents)
function cleanDir(dir) {
if (fs.existsSync(dir)) {
for (const file of fs.readdirSync(dir)) {
if (file !== '.gitignore') {
const filePath = path.join(dir, file);
if (fs.lstatSync(filePath).isDirectory()) {
fs.rmSync(filePath, { recursive: true, force: true });
} else {
fs.unlinkSync(filePath);
}
}
}
} else {
fs.mkdirSync(dir, { recursive: true });
}
}
// Convert SVG to PNG with transparency
function svgToPng(svgBuffer, width, height) {
// No background specified to maintain transparency
const resvg = new Resvg(svgBuffer, {
fitTo: { mode: 'width', value: width || 256 }
});
const pngData = resvg.render().asPng();
return pngData;
}
// Convert SVG to JPG
function svgToJpg(svgBuffer, width, height) {
// For JPGs we need a white background since JPG doesn't support transparency
const resvg = new Resvg(svgBuffer, {
background: 'white',
fitTo: { mode: 'width', value: width || 256 }
});
const pngData = resvg.render().asPng();
return pngData;
}
// Pregenerate PNG and JPG images for SVG files
function pregenerateImages(logoFiles, imagesDir, imagesVarDir) {
cleanDir(imagesVarDir);
// Only process SVG files
const svgFiles = logoFiles.filter(file => /\.svg$/i.test(file));
for (const file of svgFiles) {
const base = getBaseName(file);
const svgPath = path.join(imagesDir, file);
// Validate and fix SVG before processing
validateAndFixSvg(svgPath);
const pngPath = path.join(imagesVarDir, base + '.png');
const jpgPath = path.join(imagesVarDir, base + '.jpg');
try {
const svgBuffer = fs.readFileSync(svgPath);
const pngBuffer = svgToPng(svgBuffer, 256, 256);
fs.writeFileSync(pngPath, pngBuffer);
const jpgBuffer = svgToJpg(svgBuffer);
fs.writeFileSync(jpgPath, jpgBuffer);
} catch (e) {
console.error('Error generating PNG/JPG for', file, e);
}
}
}
// Scan directory and update logo objects
function scanLogos() {
console.log(`Scanning logos directory: ${imagesDir}`);
let existing = [];
if (fs.existsSync(outputFile)) {
try {
existing = JSON.parse(fs.readFileSync(outputFile, 'utf8'));
} catch (e) {
console.error('Could not parse existing logos.json:', e);
}
}
const collectionName = collectionArg ? collectionArg.split('=')[1] : (process.env.COLLECTION || 'all');
// Execute a script with proper error handling
function runScript(scriptName, collection) {
try {
if (!fs.existsSync(imagesDir)) {
console.error(`Directory does not exist: ${imagesDir}`);
return [];
}
const files = fs.readdirSync(imagesDir);
// Filter for image files (svg, png, jpg, jpeg)
const logoFiles = files.filter(file =>
/\.(svg|png|jpg|jpeg)$/i.test(file)
);
// Create a Set of all logo filenames in the directory
const logoFilesSet = new Set(logoFiles);
// Mark existing records as disabled if they are not found in the directory
for (const logo of existing) {
// Fix: If logo.path contains a slash, strip to filename only
if (logo.path.includes('/')) {
logo.path = logo.path.split('/').pop();
}
if (!logoFilesSet.has(logo.path)) {
logo.disable = true;
} else if (logo.disable) {
logo.disable = false;
}
}
// Create a Set of existing filenames to avoid duplication
const existingPathsSet = new Set(existing.map(logo => logo.path));
// Create new minimal logo objects for files that don't have records yet
const newLogos = logoFiles
.filter(file => !existingPathsSet.has(file))
.map(file => {
const format = getFileExtension(file);
// Only add minimal fields for new files
return {
name: formatName(file),
path: file,
format: format,
disable: false
};
})
.sort((a, b) => a.name.localeCompare(b.name));
// Merge existing and new logos (add new at the end)
let merged = [...existing, ...newLogos];
return merged;
const scriptPath = path.join(__dirname, scriptName);
const command = `node "${scriptPath}" --collection=${collection}`;
console.log(`\n=== Running ${scriptName} for ${collection} ===`);
execSync(command, { stdio: 'inherit' });
console.log(`=== Completed ${scriptName} ===`);
} catch (error) {
console.error('Error scanning logos directory:', error);
return [];
console.error(`Error running ${scriptName}:`, error.message);
process.exit(1);
}
}
// Save logos data to JSON file
function saveLogosToJson(logos) {
try {
const data = JSON.stringify(logos, null, 2);
fs.writeFileSync(outputFile, data);
console.log(`Successfully wrote ${logos.length} logos to ${outputFile}`);
} catch (error) {
console.error('Error writing logos data to file:', error);
}
}
// SVG validation and fixing function
function validateAndFixSvg(svgPath) {
try {
let svgContent = fs.readFileSync(svgPath, 'utf8');
let modified = false;
// Clean up SVG content
const originalContent = svgContent;
// Remove XML declaration
svgContent = svgContent.replace(/<\?xml[^>]*\?>\s*/gi, '');
// Remove DOCTYPE declaration
svgContent = svgContent.replace(/<!DOCTYPE[^>]*>\s*/gi, '');
// Remove comments
svgContent = svgContent.replace(/<!--[\s\S]*?-->/g, '');
// Remove leading/trailing whitespace and ensure it starts with <svg
svgContent = svgContent.trim();
if (originalContent !== svgContent) {
modified = true;
console.log(`${path.basename(svgPath)}: Cleaned up SVG (removed XML/DOCTYPE/comments)`);
}
// Parse SVG tag attributes
const svgTagMatch = svgContent.match(/<svg[^>]*>/i);
if (!svgTagMatch) {
console.warn(`No SVG tag found in ${path.basename(svgPath)}`);
return;
}
const svgTag = svgTagMatch[0];
const viewBoxMatch = svgTag.match(/viewBox\s*=\s*["']([^"']+)["']/i);
const widthMatch = svgTag.match(/width\s*=\s*["']([^"']+)["']/i);
const heightMatch = svgTag.match(/height\s*=\s*["']([^"']+)["']/i);
const hasViewBox = !!viewBoxMatch;
const hasWidth = !!widthMatch;
const hasHeight = !!heightMatch;
const width = hasWidth ? widthMatch[1] : null;
const height = hasHeight ? heightMatch[1] : null;
if (!hasViewBox && !hasWidth && !hasHeight) {
console.warn(`${path.basename(svgPath)}: No viewBox, width, or height found - cannot determine dimensions`);
return;
}
let newSvgTag = svgTag;
if (!hasViewBox && hasWidth && hasHeight) {
// Add viewBox using width and height
const widthValue = parseFloat(width);
const heightValue = parseFloat(height);
if (!isNaN(widthValue) && !isNaN(heightValue)) {
const viewBoxValue = `0 0 ${widthValue} ${heightValue}`;
newSvgTag = newSvgTag.replace(/(<svg[^>]*?)>/i, `$1 viewBox="${viewBoxValue}">`);
modified = true;
console.log(`${path.basename(svgPath)}: Added viewBox="${viewBoxValue}"`);
}
}
// Update width and height to 100% if they exist
if (hasWidth && width !== '100%') {
newSvgTag = newSvgTag.replace(/width\s*=\s*["'][^"']+["']/i, 'width="100%"');
modified = true;
console.log(`${path.basename(svgPath)}: Updated width to 100%`);
}
if (hasHeight && height !== '100%') {
newSvgTag = newSvgTag.replace(/height\s*=\s*["'][^"']+["']/i, 'height="100%"');
modified = true;
console.log(`${path.basename(svgPath)}: Updated height to 100%`);
}
if (modified) {
svgContent = svgContent.replace(svgTag, newSvgTag);
fs.writeFileSync(svgPath, svgContent, 'utf8');
console.log(`${path.basename(svgPath)}: SVG file updated`);
}
} catch (error) {
console.error(`Error processing SVG ${path.basename(svgPath)}:`, error.message);
}
}
// Main function
// Main batch processing function
function main() {
// If no collection is specified, process all collections
if (!collectionArg && !process.env.COLLECTION) {
for (const col of collections) {
const imagesDir = path.join(__dirname, '..', 'public', col.baseDir);
const outputFile = path.join(__dirname, '..', 'public', col.dataFile);
const varDir = path.join(__dirname, '..', 'public', col.varDir);
if (!fs.existsSync(imagesDir)) {
fs.mkdirSync(imagesDir, { recursive: true });
}
const files = fs.readdirSync(imagesDir);
// Only update/disable/add, do not overwrite existing keys
let existing = [];
if (fs.existsSync(outputFile)) {
try {
existing = JSON.parse(fs.readFileSync(outputFile, 'utf8'));
} catch (e) {
console.error('Could not parse existing', col.dataFile + ':', e);
}
}
// Filter for image files (svg, png, jpg, jpeg)
const logoFiles = files.filter(file =>
/\.(svg|png|jpg|jpeg)$/i.test(file)
);
const logoFilesSet = new Set(logoFiles);
// Validate and fix SVG files
const svgFiles = logoFiles.filter(file => /\.svg$/i.test(file));
for (const file of svgFiles) {
const svgPath = path.join(imagesDir, file);
validateAndFixSvg(svgPath);
}
for (const logo of existing) {
// Fix: If logo.path contains a slash, strip to filename only
if (logo.path.includes('/')) {
logo.path = logo.path.split('/').pop();
}
if (!logoFilesSet.has(logo.path)) {
logo.disable = true;
} else if (logo.disable) {
logo.disable = false;
}
}
const existingPathsSet = new Set(existing.map(logo => logo.path));
const newLogos = logoFiles
.filter(file => !existingPathsSet.has(file))
.map(file => {
const format = getFileExtension(file);
return {
name: formatName(file),
path: file,
format: format,
disable: false
};
})
.sort((a, b) => a.name.localeCompare(b.name));
let merged = [...existing, ...newLogos];
pregenerateImages(files, imagesDir, varDir);
try {
const data = JSON.stringify(merged, null, 2);
fs.writeFileSync(outputFile, data);
console.log(`Successfully wrote ${merged.length} items to ${outputFile}`);
} catch (error) {
console.error('Error writing data to file:', outputFile, error);
}
}
console.log('🚀 Starting data update process...');
if (collectionName === 'all') {
console.log('Processing all collections');
} else {
// Single collection mode (as before)
const logos = scanLogos();
const files = fs.readdirSync(imagesDir);
pregenerateImages(files, imagesDir, varDir);
saveLogosToJson(logos);
const collection = collections.find(c => c.name === collectionName);
if (!collection) {
console.error(`Collection "${collectionName}" not found`);
process.exit(1);
}
console.log(`Processing collection: ${collection.label}`);
}
// Step 1: Clean and validate SVG files
console.log('\n📋 Step 1: SVG Cleanup and Validation');
runScript('svg-cleanup.js', collectionName);
// Step 2: Generate PNG/JPG variants
console.log('\n🖼 Step 2: Generate Image Variants');
runScript('generate-variants.js', collectionName);
// Step 3: Sync data files with filesystem
console.log('\n📄 Step 3: Sync Data Files');
runScript('sync-data.js', collectionName);
// Step 4: Generate PWA cache list
console.log('\n💾 Step 4: Generate PWA Cache List');
runScript('generate-pwa-cache-list.js', 'all');
console.log('\n✅ All tasks completed successfully!');
}
// Run the script
// Run the batch process
main();