dataminer/largefetch.js

const fetch = require("node-fetch");
var mysql = require('mysql');
const TelegramBot = require('node-telegram-bot-api');

const simCo = require('./simco-credentials');
const tgInfo = require('./telegram-credentials');
if (!simCo.cookie || !tgInfo.token || !tgInfo.debugChat) {
    console.error("ERR: credentials not set.")
    return;
}

var connection = mysql.createConnection({
    host: 'localhost',
    user: 'simcompanies',
    password: '',
    database: 'simcompanies'
});
const tgBot = new TelegramBot(tgInfo.token, { polling: false });
tgBot.sendMessage(tgInfo.debugChat, 'INF: dataminer started polling resource prices.');

const exchangeAPI = "https://www.simcompanies.com/api/market/";
const resourceAPI = "https://www.simcompanies.com/api/v3/en/encyclopedia/resources/";

const actualdate = new Date();

const getFullData = async url => {

    var resourceList = await fetch(resourceAPI);
    resourceList = await resourceList.json();
    let finalData = [];
    let marketCap = [];
    for (let t1 = 0; t1 < resourceList.length; t1++) {
        console.log(`fetching resource ${t1 + 1}/${resourceList.length}`)
        var marketData = await fetch(exchangeAPI + resourceList[t1].db_letter, {
            headers: {
                Cookie: simCo.cookie
            }
        });
        if (marketData.status === 200) {
            marketData = await marketData.json();
            let foundQ = [false, false, false, false, false, false, false, false];
            let mCapSum = [0, 0, 0, 0, 0, 0, 0, 0];
            let mCapCount = [0, 0, 0, 0, 0, 0, 0, 0];
            let res = [];
            for (let t2 = 0; t2 < marketData.length; t2++) {
                if (marketData[t2].quality <= 7) {
                    if (!foundQ[marketData[t2].quality]) {
                        foundQ[marketData[t2].quality] = true;
                        res.push(marketData[t2]);
                    }
                    mCapSum[marketData[t2].quality] += marketData[t2].price * marketData[t2].quantity;
                    mCapCount[marketData[t2].quality] += marketData[t2].quantity;
                }
            }
            for (let t3 = 0; t3 < res.length; t3++) {
                finalData.push(res[t3]);

                marketCap.push({
                    kind: resourceList[t1].db_letter,
                    quality: t3,
                    avgprice: mCapCount[t3] === 0 ? 0 : mCapSum[t3] / mCapCount[t3],
                    count: mCapCount[t3],
                })
            }
        } else {
            tgBot.sendMessage(tgInfo.debugChat, `ERROR: fetching market:\nStatus Code: ${marketData.status}`);
            throw ("fetch returned status code " + marketData.status);
        }
    }
    return { data: finalData, cap: marketCap };
}

const storeFullData = async marketData => {
    marketData = await marketData;
    marketCap = marketData["cap"];
    marketData = marketData["data"];

    connection.beginTransaction(function (err) {
        if (err) { throw err; }
        for (var data in marketData) {
            const kind = mysql.escape(marketData[data]["kind"]);
            const price = mysql.escape(marketData[data]["price"]);
            const quality = mysql.escape(marketData[data]["quality"]);
            const storedate = `FROM_UNIXTIME(${Math.round(actualdate.valueOf() / 1000)})`;
            const querystring = `INSERT INTO marketv2 (kind,price,time, quality) VALUES (${kind},${price},${storedate}, ${quality})`
            connection.query(querystring, function (error, results, fields) {
                if (error) {
                    tgBot.sendMessage(tgInfo.debugChat, `ERROR: inserting marketdata - database error:\n${error}\nQUERY: ${querystring}`);
                    return connection.rollback(function () {
                        throw error;
                    });
                }
            });
        }

        for (var data in marketCap) {
            const kind = mysql.escape(marketCap[data]["kind"]);
            const price = mysql.escape(marketCap[data]["avgprice"]);
            const quality = mysql.escape(marketCap[data]["quality"]);
            const amount = mysql.escape(marketCap[data]["count"]);
            const storedate = `FROM_UNIXTIME(${Math.round(actualdate.valueOf() / 1000)})`;
            const querystring = `INSERT INTO marketcap (kind,quality,price,amount,time) VALUES (${kind},${quality},${price},${amount},${storedate})`;
            connection.query(querystring, function (error, results, fields) {
                if (error) {
                    tgBot.sendMessage(tgInfo.debugChat, `ERROR: inserting marketcaps - database error:\n${error}\nQUERY: ${querystring}`);
                    return connection.rollback(function () {
                        throw error;
                    });
                }
            });
        }

        connection.commit(function (err) {
            if (err) {
                tgBot.sendMessage(tgInfo.debugChat, 'ERROR: commit failed.');
                return connection.rollback(function () {
                    connection.end();
                    throw err;
                });
            } else {
                tgBot.sendMessage(tgInfo.debugChat, 'INF: dataminer successfully stored new prices.');
                return connection.end();
            }
        });
    });
}

storeFullData(getFullData());