Skip to content

Commit

Permalink
Move downloaded fixtures directory
Browse files Browse the repository at this point in the history
  • Loading branch information
omrilotan committed May 23, 2024
1 parent fc67031 commit e77c4d1
Show file tree
Hide file tree
Showing 11 changed files with 38,274 additions and 15 deletions.
2 changes: 1 addition & 1 deletion .gitattributes
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
* text eol=lf

fixtures/downloads/* linguist-vendored
fixtures/downloaded/* linguist-vendored
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ npm run prepare -- -f

- Manual legit browsers list: [fixtures/browsers.yml](https://github.com/omrilotan/isbot/blob/main/fixtures/browsers.yml)
- Manual known crawlers list: [fixtures/crawlers.yml](https://github.com/omrilotan/isbot/blob/main/fixtures/crawlers.yml)
- Downloaded resources end up in [fixtures/downloads](https://github.com/omrilotan/isbot/blob/main/fixtures/downloads) folder
- Downloaded resources end up in [fixtures/downloaded](https://github.com/omrilotan/isbot/blob/main/fixtures/downloaded) folder
- [user-agents](https://www.npmjs.com/package/user-agents) package is used to add randomly generated browser user agents

### Misidentification
Expand Down
1 change: 1 addition & 0 deletions fixtures/downloaded/downloaded
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fri, 10 May 2024 12:22:50 GMT
203 changes: 203 additions & 0 deletions fixtures/downloaded/kikobeats.json

Large diffs are not rendered by default.

931 changes: 931 additions & 0 deletions fixtures/downloaded/matomo-org.json

Large diffs are not rendered by default.

1,149 changes: 1,149 additions & 0 deletions fixtures/downloaded/monperrus.json

Large diffs are not rendered by default.

10,246 changes: 10,246 additions & 0 deletions fixtures/downloaded/myip.ms.json

Large diffs are not rendered by default.

25,730 changes: 25,730 additions & 0 deletions fixtures/downloaded/user-agents.net.json

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion fixtures/downloads
Submodule downloads deleted from d9d8e3
14 changes: 7 additions & 7 deletions scripts/prepare/build/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ const readFixturesYaml = async (path) =>
/**
* Build the lists of user agent strings
* @param {string} fixturesDirectory
* @param {string} downloadsDirectory
* @param {string} downloadedDirectory
* @returns {Promise<{browsers: string[], crawlers: string[]}>
*/
export async function build({ fixturesDirectory, downloadsDirectory }) {
export async function build({ fixturesDirectory, downloadedDirectory }) {
return {
browsers: Array.from(new Set(await browsers({ fixturesDirectory }))).sort(),
crawlers: Array.from(
new Set(await crawlers({ fixturesDirectory, downloadsDirectory })),
new Set(await crawlers({ fixturesDirectory, downloadedDirectory })),
).sort(),
};
}
Expand All @@ -37,21 +37,21 @@ async function browsers({ fixturesDirectory }) {
/**
* List of known crawlers user agent strings
* @param {string} fixturesDirectory
* @param {string} downloadsDirectory
* @param {string} downloadedDirectory
* @returns {string[]}
*/
async function crawlers({ fixturesDirectory, downloadsDirectory }) {
async function crawlers({ fixturesDirectory, downloadedDirectory }) {
const crawlers = await readFixturesYaml(
join(fixturesDirectory, "crawlers.yml"),
);
const browsersList = await browsers({ fixturesDirectory });
const downloaded = [];
for (const file of await readdir(downloadsDirectory)) {
for (const file of await readdir(downloadedDirectory)) {
if (!file.endsWith(".json")) {
continue;
}
try {
const content = await readFile(join(downloadsDirectory, file));
const content = await readFile(join(downloadedDirectory, file));
downloaded.push(...JSON.parse(content.toString()));
} catch (error) {
// Ignore
Expand Down
10 changes: 5 additions & 5 deletions scripts/prepare/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ start(process);
async function start({ argv }) {
const { force } = args({ argv });
const fixturesDirectory = join("fixtures");
const downloadsDirectory = join(fixturesDirectory, "downloads");
const downloadedDirectory = join(fixturesDirectory, "downloaded");

await mkdir(downloadsDirectory, { recursive: true });
const results = await download({ dir: downloadsDirectory, force });
await mkdir(downloadedDirectory, { recursive: true });
const results = await download({ dir: downloadedDirectory, force });
const news = results.reduce((a, b) => a + b);
if (news) {
log("Create new timestamp");
await writeFile(
join(downloadsDirectory, "downloaded"),
join(downloadedDirectory, "downloaded"),
new Date().toUTCString(),
);
} else {
Expand All @@ -39,7 +39,7 @@ async function start({ argv }) {
log("Create fixtures JSON");
const { browsers, crawlers } = await build({
fixturesDirectory,
downloadsDirectory,
downloadedDirectory,
});
await writeFile(
join(fixturesDirectory, "index.json"),
Expand Down

0 comments on commit e77c4d1

Please sign in to comment.