Files
dataminer/largefetch.js

93 lines
3.2 KiB
JavaScript

const fetch = require("node-fetch");
var mysql = require('mysql');
const TelegramBot = require('node-telegram-bot-api');
const simCo = require('./simco-credentials');
const tgInfo = require('./telegram-credentials');
if (!simCo.cookie || !tgInfo.token) {
console.error("ERR: credentials not set.")
return;
}
var connection = mysql.createConnection({
host: 'localhost',
user: 'simcompanies',
password: '',
database: 'simcompanies'
});
const tgBot = new TelegramBot(tgInfo.token, { polling: false });
bot.sendMessage(tgInfo.debugChat, 'INF: started polling resource prices.');
const exchangeAPI = "https://www.simcompanies.com/api/market/";
const resourceAPI = "https://www.simcompanies.com/api/v3/en/encyclopedia/resources/";
const actualdate = new Date();
const getFullData = async url => {
var resourceList = await fetch(resourceAPI);
resourceList = await resourceList.json();
let finalData = [];
for (let t1 = 0; t1 < resourceList.length; t1++) {
console.log(`fetching resource ${t1 + 1}/${resourceList.length}`)
var marketData = await fetch(exchangeAPI + resourceList[t1].db_letter, {
headers: {
Cookie: simCo.cookie
}
});
if (marketData.status === 200) {
marketData = await marketData.json();
let foundQ = [false, false, false, false, false, false, false];
let res = [];
for (let t2 = 0; t2 < marketData.length; t2++) {
if (!foundQ[marketData[t2].quality]) {
foundQ[marketData[t2].quality] = true;
res.push(marketData[t2]);
}
}
for (let t3 = 0; t3 < res.length; t3++) {
finalData.push(res[t3]);
}
} else {
throw ("fetch returned status code " + marketData.status);
}
}
return finalData;
}
const storeFullData = async marketData => {
marketData = await marketData;
connection.beginTransaction(function (err) {
if (err) { throw err; }
for (var data in marketData) {
const kind = mysql.escape(marketData[data]["kind"]);
const price = mysql.escape(marketData[data]["price"]);
const quality = mysql.escape(marketData[data]["quality"]);
const storedate = `FROM_UNIXTIME(${Math.round(actualdate.valueOf() / 1000)})`;
const querystring = `INSERT INTO marketv2 (kind,price,time, quality) VALUES (${kind},${price},${storedate}, ${quality})`
connection.query(querystring, function (error, results, fields) {
if (error) {
return connection.rollback(function () {
throw error;
});
}
});
}
connection.commit(function (err) {
if (err) {
return connection.rollback(function () {
connection.end();
throw err;
});
} else {
connection.end();
}
});
});
bot.sendMessage(tgInfo.debugChat, 'INF: successfully stored new prices.');
}
storeFullData(getFullData());