add large fetch function

fetching all lowest prices for all qualities of all products

largefetch.js

@@ -0,0 +1,81 @@
+const fetch = require("node-fetch");
+var mysql = require('mysql');
+var connection = mysql.createConnection({
+    host: 'localhost',
+    user: 'simcompanies',
+    password: '',
+    database: 'simcompanies'
+});
+const simCo = require('./credentials');
+
+const exchangeAPI = "https://www.simcompanies.com/api/market/";
+const resourceAPI = "https://www.simcompanies.com/api/v3/en/encyclopedia/resources/";
+
+const actualdate = new Date();
+
+const getFullData = async url => {
+
+    var resourceList = await fetch(resourceAPI);
+    resourceList = await resourceList.json();
+    let finalData = [];
+    for (let t1 = 0; t1 < resourceList.length; t1++) {
+        console.log(`fetching resource ${t1 + 1}/${resourceList.length}`)
+        var marketData = await fetch(exchangeAPI + resourceList[t1].db_letter, {
+            headers: {
+                Cookie: simCo.cookie
+            }
+        });
+        if (marketData.status === 200) {
+            marketData = await marketData.json();
+            let foundQ = [false, false, false, false, false, false, false];
+            let res = [];
+            for (let t2 = 0; t2 < marketData.length; t2++) {
+                if (!foundQ[marketData[t2].quality]) {
+                    foundQ[marketData[t2].quality] = true;
+                    res.push(marketData[t2]);
+                }
+            }
+            for (let t3 = 0; t3 < res.length; t3++) {
+                finalData.push(res[t3]);
+            }
+        } else {
+            throw ("fetch returned status code " + marketData.status);
+        }
+    }
+    return finalData;
+}
+
+const storeFullData = async marketData => {
+    marketData = await marketData;
+
+    connection.beginTransaction(function (err) {
+        if (err) { throw err; }
+        for (var data in marketData) {
+            const kind = mysql.escape(marketData[data]["kind"]);
+            const price = mysql.escape(marketData[data]["price"]);
+            const quality = mysql.escape(marketData[data]["quality"]);
+            const storedate = `FROM_UNIXTIME(${Math.round(actualdate.valueOf() / 1000)})`;
+            const querystring = `INSERT INTO marketv2 (kind,price,time, quality) VALUES (${kind},${price},${storedate}, ${quality})`
+            connection.query(querystring, function (error, results, fields) {
+                if (error) {
+                    return connection.rollback(function () {
+                        throw error;
+                    });
+                }
+            });
+        }
+
+        connection.commit(function (err) {
+            if (err) {
+                return connection.rollback(function () {
+                    connection.end();
+                    throw err;
+                });
+            } else {
+                connection.end();
+            }
+        });
+    });
+}
+
+storeFullData(getFullData());