blowfish/processUsers.js

149 lines
4.1 KiB
JavaScript
Raw Permalink Normal View History

2023-07-14 05:12:21 -05:00
const fs = require('fs');
2024-11-29 10:45:32 -06:00
const crypto = require("crypto")
2023-07-14 05:12:21 -05:00
const puppeteer = require("puppeteer");
const configDir = "./exampleSite/config/_default";
const defaultLang = "en";
2023-07-14 05:12:21 -05:00
const usersFolderPath = "./exampleSite/content/users/"
2024-03-13 19:01:32 -05:00
const translate = require('@iamtraction/google-translate');
2023-07-14 05:12:21 -05:00
var targetLangs = []
const configFiles = fs.readdirSync(configDir);
configFiles.forEach(file => {
2024-11-29 10:45:32 -06:00
if (file.indexOf("languages.") > -1) {
var lang = file.split(".")[1];
if (lang != defaultLang) {
targetLangs.push(lang);
}
2024-11-29 10:45:32 -06:00
}
});
const indexFiles = fs.readdirSync(usersFolderPath);
2024-03-13 19:01:32 -05:00
for (var i in targetLangs) {
2024-11-29 10:45:32 -06:00
var targetFile = '_index.' + targetLangs[i] + '.md';
if (indexFiles.indexOf(targetFile) == -1) {
fs.copyFileSync(usersFolderPath + '_index.md', usersFolderPath + targetFile);
}
}
2023-07-14 05:12:21 -05:00
let rawdata = fs.readFileSync(usersFolderPath + 'users.json');
let users = JSON.parse(rawdata);
2023-07-14 06:37:16 -05:00
let userDict = {}
2023-07-20 15:34:37 -05:00
for (var i in users) {
2024-11-29 10:45:32 -06:00
userDict[generateDirName(users[i].url)] = true;
2023-07-14 06:37:16 -05:00
}
2023-07-14 05:12:21 -05:00
2023-07-14 05:28:12 -05:00
const files = fs.readdirSync(usersFolderPath);
2023-07-14 06:37:16 -05:00
2023-07-14 05:12:21 -05:00
2023-07-14 05:28:12 -05:00
for (file in files) {
2024-11-29 10:45:32 -06:00
let stats = fs.statSync(usersFolderPath + files[file]);
if (files[file] != 'users.json' && files[file].indexOf("_index.") == -1) {
if (stats.isDirectory()) {
if (!userDict[files[file]]) {
console.log('deleting: ', files[file]);
fs.rmdirSync(usersFolderPath + files[file], { recursive: true, force: true });
}
} else {
console.log('deleting: ', files[file]);
fs.unlinkSync(usersFolderPath + files[file]);
2023-07-14 05:28:12 -05:00
}
2024-11-29 10:45:32 -06:00
}
2023-07-14 05:28:12 -05:00
}
2023-07-14 05:12:21 -05:00
2024-03-13 19:01:32 -05:00
var cache = {};
2024-11-29 10:45:32 -06:00
function generateDirName(seed) {
var hash = crypto.createHash('md5');
hash.update(seed);
var dir = hash.digest('hex');
return dir;
}
2024-03-13 19:01:32 -05:00
async function convert(text, from, to) {
2024-11-29 10:45:32 -06:00
var options = {
from: from,
to: to
};
if (!cache[to])
cache[to] = {};
if (cache[to][text])
return cache[to][text];
var translated_text = await translate(text, options);
cache[to][text] = translated_text.text;
return translated_text.text;
2024-03-13 19:01:32 -05:00
}
async function translateFrontMatterTags(block, targetLang, tags) {
2024-11-29 10:45:32 -06:00
var array = block.split("\n");
var translatedBlock = "";
for (var i = 0; i < array.length; i++) {
if (array[i].indexOf(":") > -1) {
var elements = array[i].split(":");
var newElement = "";
if (elements[0].indexOf("tags") != -1) {
translatedTags = []
for (var j in tags) {
var tempTag = await convert(tags[j], defaultLang, targetLang);
translatedTags.push(tempTag);
2024-03-13 19:01:32 -05:00
}
2024-11-29 10:45:32 -06:00
var trasnlatedTagsString = translatedTags.join(", ");
newElement = elements[0] + ": [" + trasnlatedTagsString + "]";
} else {
newElement = array[i];
}
} else {
newElement = array[i];
2024-03-13 19:01:32 -05:00
}
2024-11-29 10:45:32 -06:00
translatedBlock += newElement + "\n";
}
return translatedBlock;
2024-03-13 19:01:32 -05:00
}
2023-07-14 05:12:21 -05:00
2023-07-14 05:28:12 -05:00
puppeteer
2024-11-29 10:45:32 -06:00
.launch({
defaultViewport: {
width: 1280,
height: 800,
},
})
.then(async (browser) => {
2023-07-14 05:12:21 -05:00
2024-11-29 10:45:32 -06:00
const page = await browser.newPage();
2023-07-14 05:12:21 -05:00
2024-11-29 10:45:32 -06:00
for (var i in users) {
2023-07-14 05:28:12 -05:00
2024-11-29 10:45:32 -06:00
var userMDFile = "---\n\
2023-07-14 05:28:12 -05:00
title: \""+ users[i].title + "\"\n\
2023-07-20 15:34:37 -05:00
tags: ["+ users[i].tags + "]\n\
2023-07-14 05:28:12 -05:00
externalUrl: \""+ users[i].url + "\"\n\
2024-11-29 10:45:32 -06:00
weight: "+ (i + 1) + "\n\
2023-07-14 05:28:12 -05:00
showDate: false\n\
showAuthor: false\n\
showReadingTime: false\n\
showEdit: false\n\
showLikes: false\n\
showViews: false\n\
layoutBackgroundHeaderSpace: false\n\
\r---\n";
2024-11-29 10:45:32 -06:00
var dir = usersFolderPath + generateDirName(users[i].url);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
console.log(i, users[i].title);
fs.writeFileSync(dir + '/index.md', userMDFile);
for (var j in targetLangs) {
var content = await translateFrontMatterTags(userMDFile, targetLangs[j], users[i].tags);
fs.writeFileSync(dir + '/index.' + targetLangs[j] + '.md', content);
2023-07-14 05:28:12 -05:00
}
2024-11-29 10:45:32 -06:00
await page.goto(users[i].url);
await page.screenshot({ path: dir + "/feature.jpg" });
}
}
2023-07-14 05:28:12 -05:00
2024-11-29 10:45:32 -06:00
await browser.close();
2023-07-14 05:28:12 -05:00
2024-11-29 10:45:32 -06:00
});