Fix weapons-wiki url.

Refactor to use scribunto for most of data

Readme.md

@@ -58,10 +58,8 @@ Example return for `/weapons-wiki`:
 ```json
 {
     "data": {
-      "Augments": [{}, {}],
-      "IgnoreInCount": ["", ""],
-      "Stances": [{}, {}],
-      "Weapons": [{}, {}]
+      "Supra Vandal": {},
+      "IronBride": {},
     },
     "meta": {
         "nRefresh": 1,

cache.js

@@ -20,8 +20,10 @@
 "use strict";
 const https = require("https");
 const crypto = require("crypto");
-const getLuaObject = require("./lua/getLuaObject");
+const { getData, getDataNoClean } = require("./lua/getLuaObject");
 const CustomError = require("./error").CustomError;
+const getLuaObject = getData;
+const getLuaObjectNoClean = getDataNoClean;
 
 /**
  * @typedef {Object} MetaObject
@@ -62,9 +64,11 @@ class Cache {
     this.dataUrl = dataUrl;
     this.dependentOn = dependentOn;
 
-    const dependencyResolve = this.dependentOn ? this.dependentOn.map((dependency) => {
-      return dependency.cache.get();
-    }) : []
+    const dependencyResolve = this.dependentOn
+      ? this.dependentOn.map((dependency) => {
+          return dependency.cache.get();
+        })
+      : [];
     Promise.all(dependencyResolve)
       .then(() => {
         return this.get();
@@ -224,4 +228,157 @@ class Cache {
   }
 }
 
-module.exports = Cache;
+/**
+ * A cache module class.
+ */
+class CacheNew {
+  /**
+   * Create a new cache module.
+   * @param {string} dataUrl Url to the Wikia module.
+   * @param {string} name Name of the cache module.
+   */
+  constructor(dataUrl, name) {
+    this.data = null;
+    this.src = null;
+    this.name = name;
+    this.lastRefresh = 0;
+    this.nRefresh = 0;
+    this.dataUrl = dataUrl;
+
+    this.get()
+      .then(() => {
+        console.log("Cache initialized for: " + this.name);
+      })
+      .catch((e) => {
+        console.log("Failed to initialize cache for: " + this.name);
+        console.error(e);
+      });
+  }
+
+  /**
+   * Make a 'GET' request to given Wikia url and get the data inside of the lua object.
+   * @param {string} url Wikia api url.
+   * @returns {Promise<object, Object>} The content of the lua object.
+   * @private
+   */
+  _getRequest(url) {
+    return new Promise((resolve, reject) => {
+      https
+        .get(url, (res) => {
+          let data = "";
+          res.on("data", (d) => {
+            data += d;
+          });
+
+          res.on("end", () => {
+            if (data.length === 0) {
+              return reject(new CustomError("No data."));
+            }
+            try {
+              let parsed = JSON.parse(data.toString());
+              this.src = parsed.return;
+              getLuaObjectNoClean(this.src)
+                .then((data) => {
+                  return resolve(data);
+                })
+                .catch((err) => {
+                  return reject(
+                    new CustomError("Failed to get lua data.", err)
+                  );
+                });
+            } catch (e) {
+              return reject(
+                new CustomError("Failed to parse or find revisions.", e)
+              );
+            }
+          });
+        })
+        .on("error", (err) => {
+          return reject(new CustomError("Failed to retrieve data.", err));
+        });
+    });
+  }
+
+  /**
+   * Create a sha256 hash of the cached data.
+   * @returns {PromiseLike<ArrayBuffer>} Base64 encoded hash.
+   */
+  createHash() {
+    let hash = crypto.createHash("sha256");
+    hash.update(JSON.stringify(this.data));
+    return hash.digest("base64");
+  }
+
+  /**
+   * Create a meta data object.
+   * @returns {MetaObject}
+   */
+  createMeta() {
+    return {
+      nRefresh: this.nRefresh,
+      lastRefresh: this.lastRefresh,
+      hash: this.createHash(),
+      license: "CC BY-SA",
+    };
+  }
+
+  /**
+   * Get the cached data or refresh the cache.
+   * @returns {Promise<ResponseObject,Object>} Returns the cached data object.
+   */
+  get() {
+    return new Promise((resolve, reject) => {
+      if (
+        !this.data ||
+        !this.lastRefresh ||
+        this.lastRefresh + 1000 * 60 * 60 < Date.now()
+      ) {
+        if (this.dataUrl) {
+          this._getRequest(this.dataUrl)
+            .then((data) => {
+              this.lastRefresh = Date.now();
+              this.data = data;
+              this.nRefresh++;
+              return resolve({
+                data: this.data,
+                meta: this.createMeta(),
+              });
+            })
+            .catch((err) => {
+              return reject(new CustomError("Failed at requesting data.", err));
+            });
+        } else {
+          return reject(
+            new CustomError("No data url.", {}, { name: this.name })
+          );
+        }
+      } else {
+        return resolve({
+          data: this.data,
+          meta: this.createMeta(),
+        });
+      }
+    });
+  }
+
+  /**
+   * Get the meta data for cache module
+   * @returns {Promise<MetaObject, Object>} The metadata object for cache module.
+   */
+  getMeta() {
+    return new Promise((resolve, reject) => {
+      this.get()
+        .then(() => {
+          return resolve({ meta: this.createMeta() });
+        })
+        .catch((e) => {
+          return reject(new CustomError("Failed to generate meta object.", e));
+        });
+    });
+  }
+}
+
+module.exports = {
+  Cache,
+  CacheNew,
+};

lua/getLuaObject.js

@@ -47,8 +47,14 @@ function getData(input, dependencies) {
     input = input.replace(replaceModuleTable, "require('module_table')");
 
     // Special fixes to weapon data table... I expect these to break rather soon.....
-    input = input.replace(`local attack = weaponEntry['Attack'..i] or weaponEntry.attack[i]`, `local attack = weaponEntry['Attack'..i]`)
-    input = input.replace(`weaponEntry['Attack'..i], weaponEntry.attack[i] = attack, attack`, `weaponEntry['Attack'..i] = attack`)
+    input = input.replace(
+      `local attack = weaponEntry['Attack'..i] or weaponEntry.attack[i]`,
+      `local attack = weaponEntry['Attack'..i]`
+    );
+    input = input.replace(
+      `weaponEntry['Attack'..i], weaponEntry.attack[i] = attack, attack`,
+      `weaponEntry['Attack'..i] = attack`
+    );
 
     // dependency to "Warframes" cache from "WarframesConclave" cache
     if (dependencies) {
@@ -58,7 +64,7 @@ function getData(input, dependencies) {
         const dependency = dependencies.find((v) => v.module === result[1]);
         input = input.replace(
           `mw.loadData('${dependency.module}')`,
-          dependency.cache.src.replace('return', '')
+          dependency.cache.src.replace("return", "")
         );
       }
     }
@@ -121,4 +127,49 @@ end
   });
 }
 
-module.exports = getData;
+function getDataNoClean(input) {
+  return new Promise((resolve, reject) => {
+    let luaCmd = "lua5.3";
+    let luaParams = ["./luaObjectToJson.lua"];
+    if (process.platform === "win32") {
+      luaCmd = "./lua53.exe";
+    }
+    let luaToJson = spawn(luaCmd, luaParams, { cwd: __dirname });
+
+    luaToJson.stderr.pipe(process.stderr);
+    luaToJson.stdin.setEncoding("utf-8");
+    luaToJson.stdin.write(input);
+    luaToJson.stdin.end();
+
+    let data = "";
+    luaToJson.stdout.on("data", (d) => {
+      data += d;
+    });
+    let err = "";
+    luaToJson.stderr.on("data", (d) => {
+      err += d;
+    });
+    luaToJson.stderr.on("end", (d) => {
+      if (err.length > 0) {
+        return reject(err);
+      }
+    });
+    luaToJson.stdout.on("end", (c) => {
+      let d = "";
+      if (data.length === 0) {
+        return reject(new CustomError("Lua parsing failed."));
+      }
+      try {
+        d = JSON.parse(data);
+        return resolve(d);
+      } catch (e) {
+        return reject(e);
+      }
+    });
+  });
+}
+
+module.exports = {
+  getData,
+  getDataNoClean,
+};