Convert application to TypeScript

.gitignore

@@ -86,3 +86,6 @@ typings/
 
 # DynamoDB Local files
 .dynamodb/
+
+# TypeScript output
+dist/

package.json

@@ -14,8 +14,18 @@
 	"name": "reddit-dataminer",
 	"description": "Dumps Reddit scripts.",
 	"version": "1.0.0",
-	"main": "./src/index.js",
+	"main": "./dist/index.js",
 	"devDependencies": {
+		"@types/debug": "^4.1.5",
+		"@types/esprima": "^4.0.2",
+		"@types/estree": "^0.0.46",
+		"@types/fs-extra": "^9.0.8",
+		"@types/js-beautify": "^1.13.1",
+		"@types/node": "^13.9.3",
+		"@types/puppeteer": "^5.4.3",
+		"@types/string-format": "^2.0.0",
+		"@typescript-eslint/eslint-plugin": "^4.18.0",
+		"@typescript-eslint/parser": "^4.18.0",
 		"eslint": "^7.21.0",
 		"eslint-config-haykam": "^1.10.0",
 		"eslint-plugin-extra-rules": "^0.0.0-development",
@@ -24,14 +34,35 @@
 		"eslint-plugin-json": "^2.1.2",
 		"eslint-plugin-markdown": "^2.0.0",
 		"eslint-plugin-node": "^11.1.0",
-		"eslint-plugin-unicorn": "^28.0.2"
+		"eslint-plugin-unicorn": "^28.0.2",
+		"ts-node-dev": "^1.1.6",
+		"typescript": "^4.2.3"
 	},
 	"scripts": {
+		"build": "tsc",
+		"start": "node ./dist/index.js",
+		"dev": "ts-node-dev --respawn ./src/index.ts",
 		"test": "echo \"Error: no test specified\" && exit 1",
-		"lint": "eslint \"./**/*.js\" --ignore-path .gitignore"
+		"lint": "eslint \"./**/*.ts\" --ignore-path .gitignore"
 	},
 	"eslintConfig": {
-		"extends": "eslint-config-haykam"
+		"extends": [
+			"eslint-config-haykam",
+			"plugin:@typescript-eslint/eslint-recommended",
+			"plugin:@typescript-eslint/recommended"
+		],
+		"parser": "@typescript-eslint/parser",
+		"plugins": [
+			"@typescript-eslint"
+		],
+		"rules": {
+			"semi": "off",
+			"@typescript-eslint/semi": "error",
+			"no-shadow": "off",
+			"@typescript-eslint/no-shadow": "error",
+			"sort-imports": "error",
+			"jsdoc/require-param-type": "off"
+		}
 	},
 	"keywords": [
 		"reddit",

src/index.js → src/index.ts

@@ -1,19 +1,12 @@
-const { program } = require("@caporal/core");
-const { debug } = require("./util/log.js");
+import { RedditDataminerOptions, debugOpt } from "./util/options";
 
-const start = require("./start.js");
+import debug from "debug";
+import { program } from "@caporal/core";
+import start from "./start";
+import { version } from "./util/version";
 
-const { version } = require("../package.json");
 program.version(version);
 
-/**
- * @type {[string, string, import('@caporal/core').CreateOptionCommandOpts]}
- */
-const debugOpt = ["--debug [debug]", "Debuggers to enable.", {
-	default: "reddit-dataminer:*",
-	validator: program.STRING,
-}];
-
 program
 	.command("start", "Starts the script dumper.")
 	.option(...debugOpt)
@@ -57,7 +50,9 @@ program
 		default: false,
 		validator: program.BOOLEAN,
 	})
-	.action(({ options }) => {
+	.action(params => {
+		const options = params.options as unknown as RedditDataminerOptions;
+
 		debug.enable(options.debug);
 		start(options);
 	});

src/start.js → src/start.ts

@@ -1,21 +1,19 @@
-const puppeteer = require("puppeteer");
-
-const { log, dumping: dumpingLog, hashes: hashesLog } = require("./util/log.js");
-
-const fse = require("fs-extra");
-
-const { version } = require("../package.json");
+import { dumping as dumpingLog, hashes as hashesLog, log } from "./util/log";
+import puppeteer, { SetCookie } from "puppeteer";
+
+import { RedditDataminerOptions } from "./util/options";
+import dumpScripts from "./util/dump-scripts";
+import fse from "fs-extra";
+import getHashes from "./util/get-hashes";
+import getRuntimeScripts from "./util/get-runtime-scripts";
+import getScripts from "./util/get-scripts";
+import getToken from "./util/get-token";
+import { version } from "./util/version";
 
 const transformersGlobal = {
 	VERSION: version,
 };
 
-const getHashes = require("./util/get-hashes.js");
-const getScripts = require("./util/get-scripts.js");
-const getRuntimeScripts = require("./util/get-runtime-scripts.js");
-const dumpScripts = require("./util/dump-scripts.js");
-const getToken = require("./util/get-token.js");
-
 /**
  * Arguments to be passed to Puppeteer if the sandbox option is disabled.
  */
@@ -26,9 +24,9 @@ const noSandboxArgs = [
 
 /**
  * Runs the dataminer.
- * @param {Object} args The command-line arguments.
+ * @param args The command-line arguments.
  */
-async function start(args) {
+export default async function start(args: RedditDataminerOptions): Promise<string[]> {
 	log(args);
 	await fse.ensureDir(args.path);
 	log("ensured output path exists");
@@ -46,13 +44,13 @@ async function start(args) {
 	} else {
 		log("not using a reddit session token");
 	}
-	const sessionCookie = {
+	const sessionCookie: SetCookie = {
 		domain: ".reddit.com",
 		name: "reddit-session",
 		value: token,
 	};
 
-	const scriptsSet = new Set();
+	const scriptsSet: Set<string> = new Set();
 
 	const placeScripts = await getScripts(browser, hashes, sessionCookie, args.cache);
 	for (const placeScript of placeScripts) {
@@ -69,10 +67,10 @@ async function start(args) {
 	const scripts = [...scriptsSet];
 	log("got list of scripts to dump");
 
-	const transformersRun = {
+	const transformersRun: Record<string, string> = {
 		...transformersGlobal,
 		DATE: new Date().toLocaleString("en-US"),
-		SCRIPT_TOTAL: scripts.length,
+		SCRIPT_TOTAL: scripts.length + "",
 	};
 
 	if (await dumpScripts(scripts, transformersRun, args, hashes)) {
@@ -95,4 +93,3 @@ async function start(args) {
 	await browser.close();
 	return scripts;
 }
-module.exports = start;

src/util/dump-scripts.js → src/util/dump-scripts.ts

@@ -1,25 +1,23 @@
-const { dumping: log } = require("./log.js");
-const filter = require("./filter.js");
-const got = require("./got.js");
-
-const format = require("string-format");
-const beautify = require("js-beautify").js;
-
-const path = require("path");
-const fse = require("fs-extra");
+import { RedditDataminerOptions } from "./options";
+import beautify from "js-beautify";
+import { filter } from "./filter";
+import format from "string-format";
+import fse from "fs-extra";
+import { dumping as log } from "./log";
+import path from "path";
+import { uaGot } from "./got";
 
 /**
  * Dumps a single script and saves it.
- * @param {string} script The script to dump.
- * @param {RegExpMatchArray} match The match of script to dump.
- * @param {number} index The index of the script to dump.
- * @param {Object} transformersRun The run-specific transformers.
- * @param {Object} args The command-line arguments.
- * @param {Object.<string, string>} hashes The hashes for previously-saved scripts.
- * @returns {Promise<boolean>} Whether dumping was attempted for all scripts.
+ * @param script The script to dump.
+ * @param match The match of script to dump.
+ * @param index The index of the script to dump.
+ * @param transformersRun The run-specific transformers.
+ * @param args The command-line arguments.
+ * @param hashes The hashes for previously-saved scripts.
  */
-async function dumpScript(script, match, index, transformersRun, args, hashes) {
-	const response = await got(script);
+async function dumpScript(script: string, match: RegExpMatchArray, index: number, transformersRun: Record<string, string>, args: RedditDataminerOptions, hashes: Record<string, string>): Promise<void> {
+	const response = await uaGot(script);
 	const beautified = beautify(response.body, {
 		/* eslint-disable-next-line camelcase */
 		indent_with_tabs: true,
@@ -51,13 +49,13 @@ async function dumpScript(script, match, index, transformersRun, args, hashes) {
 
 /**
  * Dumps the scripts and saves them.
- * @param {string[]} scripts The scripts to dump.
- * @param {Object} transformersRun The run-specific transformers.
- * @param {Object} args The command-line arguments.
- * @param {Object.<string, string>} hashes The hashes for previously-saved scripts.
- * @returns {Promise<boolean>} Whether dumping was attempted for all scripts.
+ * @param scripts The scripts to dump.
+ * @param transformersRun The run-specific transformers.
+ * @param args The command-line arguments.
+ * @param hashes The hashes for previously-saved scripts.
+ * @returns Whether dumping was attempted for all scripts.
  */
-async function dumpScripts(scripts, transformersRun, args, hashes) {
+export default async function dumpScripts(scripts: string[], transformersRun: Record<string, string>, args: RedditDataminerOptions, hashes: Record<string, string>): Promise<boolean> {
 	// Use a for-loop to allow an early return
 	// This is not concurrent
 	if (args.stopDumpingAfterFail) {
@@ -87,4 +85,3 @@ async function dumpScripts(scripts, transformersRun, args, hashes) {
 	}));
 	return true;
 }
-module.exports = dumpScripts;

src/util/filter.ts

@@ -0,0 +1 @@
+export const filter = /https:\/\/www\.redditstatic\.com\/desktop2x\/([^.]+)\.?(.*)\.js/;

src/util/get-hash-objects.ts

@@ -0,0 +1,66 @@
+import { Node } from "estree";
+import { parseScript } from "esprima";
+
+/**
+ * Determines whether a given node is a potential object expression that stores hashes.
+ * @param node The node.
+ * @returns Whether the given node is a potential object expression that stores hashes.
+ */
+function isPotentialHashNode(node: Node): boolean {
+	return node.type === "ObjectExpression" && node.properties.length > 10;
+}
+
+/**
+ * Determines whether a given node is a `".js"` literal.
+ * @param node The node.
+ * @returns Whether the given node is a `".js"` literal.
+ */
+function isJsLiteral(node: Node): boolean {
+	return node.type === "Literal" && node.value === ".js";
+}
+
+/**
+ * Converts an object expression node to an object.
+ * @param node The object expression node.
+ * @returns The converted object.
+ */
+function convertObjectExpressionToObject(node: Node): Record<string, string> {
+	if (node.type !== "ObjectExpression") {
+		throw new TypeError("`node` parameter must be an object expression node");
+	}
+
+	const object: Record<string, string> = {};
+	for (const property of node.properties) {
+		if (property.type === "Property") {
+			const key = property.key as unknown as Record<string, string>;
+			const value = property.value as unknown as Record<string, string>;
+
+			object[key.name || key.value] = value.value;
+		}
+	}
+	return object;
+}
+
+/**
+ * Parses a program and returns potential objects that store hashes.
+ * @param program The program to parse in order to find object expression nodes.
+ * @param beforeJs Whether to only include the first object expression node preceeding a `".js"` literal, if present.
+ */
+export default function getHashObjects(program: string, beforeJs = false): Record<string, string>[] {
+	const nodes: Node[] = [];
+	parseScript(program, {}, node => {
+		nodes.push(node);
+	});
+
+	const objects = [];
+	for (const node of nodes) {
+		if (isPotentialHashNode(node)) {
+			objects.push(convertObjectExpressionToObject(node));
+		} else if (beforeJs && isJsLiteral(node) && objects.length > 0) {
+			return [
+				objects[objects.length - 1],
+			];
+		}
+	}
+	return objects;
+}