diff --git a/Readme.md b/Readme.md index 664bd04..8f903be 100644 --- a/Readme.md +++ b/Readme.md @@ -36,8 +36,10 @@ https://knowyourmeme.com/memes/this-is-fine - [planned features](#planned-features) - [Prerequisities](#prerequisities) - [Run](#run) + - [Configuration](#configuration) + - [Sleep time](#sleep-time) - [Run in Docker](#run-in-docker) - - [Download profiles](#download-profiles) + - [Run locally](#run-locally) - [examples](#examples) - [Search in result files (saved profiles)](#search-in-result-files-saved-profiles) - [examples](#examples-1) @@ -84,6 +86,28 @@ or have it in your `.zshrc` 🤗 or whatever your shell loads at start ## Run +Here's what you need to get the thing running. + +1. Configuration (optional) +2. Run in Docker +3. Run locally + +### Configuration + +Currently, the only configuration you can do is setting sleep time between request cycles. + +#### Sleep time + +**DEFAULT** sleep time is 30 seconds. + +Increase the sleep time to avoid hitting the GitHub API rate limit. + +You can customise the sleep time between cycles by setting the `SLEEP_TIME_SECONDS` environment variable. + +```bash +$ SLEEP_TIME_SECONDS=15 bb scrape +``` + ### Run in Docker All of the following should work in Docker, too. @@ -97,9 +121,11 @@ $ docker run -it --rm git-hire If you need to store the profiles, you can mount a docker volume, but this goes beyond the scope of this README. -### Download profiles +### Run locally -`$ bb scrape ` +```bash +$ bb scrape +``` Will save the github profiles as `.edn` into the `profiles` directory, **but** as GitHub support let me know: @@ -107,7 +133,9 @@ Will save the github profiles as `.edn` into the `profiles` directory, Specify further adding a language: -`$ bb scrape ` +```bash +$ bb scrape +``` **Be warned!** This might not find a PHP dev who switched to Rust recently, as described by GitHub's Support. @@ -120,7 +148,7 @@ After having built a pool of profiles, use #### examples `$ bb scrape mainz` -`$ bb scrape "Bad Schwalbach"` +`$ bb scrape "Bad Kreuznach"` `$ bb scrape wiesbaden java` `$ bb scrape wiesbaden php` `$ bb scrape mainz javascript` @@ -137,19 +165,29 @@ After having built a pool of profiles, use you might go further, by piping to bb again, unimaginable possibilities... -`$ mkdir rails; cp $(grep -Zril rails profiles) rails` +```bash +$ mkdir rails; cp $(grep -Zril rails profiles) rails +``` + +and then: -`$ bb search-keyword "ios" | bb -e '(map #(str/upper-case %) *input*)'` +```bash +$ bb search-keyword "ios" | bb -e '(map #(str/upper-case %) *input*)' +``` ### Inspect Profiles (with examples! 🤯) -`$ bb read-profile.clj simonneutert` +```bash +$ bb read-profile.clj simonneutert +``` -go further, by piping +go further, by piping: -`$ bb read-profile.clj simonneutert | bb -e '(:languages *input*)'` +```bash +$ bb read-profile.clj simonneutert | bb -e '(:languages *input*)' +``` -read many profiles +then read many profiles ```bash $ bb search-keyword ruby | bb -e '(mapv #(edn/read-string (slurp %)) *input*)' diff --git a/src/git_hire/main.clj b/src/git_hire/main.clj index 5866a0a..9bd2df3 100644 --- a/src/git_hire/main.clj +++ b/src/git_hire/main.clj @@ -27,6 +27,14 @@ (def user-search-path "/search/users") +(def default-sleep-time "30") + +(def sleep-time + (let [sleep-time (or + (System/getenv "SLEEP_TIME_SECONDS") + default-sleep-time)] + (* (Integer/parseInt sleep-time) 1000))) + (defn ->utf8 [s] (URLEncoder/encode s "UTF-8")) @@ -124,7 +132,7 @@ runs (per-page->runs total-user-count per-page) users (:items res)] (if (> total-user-count 1000) - (do (Thread/sleep (* 4 1000)) + (do (Thread/sleep (* sleep-time 1000)) (recur location lang (+ 1 more-repos-than))) (if (> runs 1) (do (prn "getting users with more than " more-repos-than " repos") @@ -147,7 +155,7 @@ runs (per-page->runs total-user-count per-page) users (:items res)] (if (> total-user-count 1000) - (do (Thread/sleep (* 4 1000)) + (do (Thread/sleep (* sleep-time 1000)) (recur location (+ 1 more-repos-than))) (do (file-path-location-all location) (if (> runs 1) @@ -161,7 +169,7 @@ (defn repo-slim [user-repo] - (select-keys user-repo [:html_url :name :description :homepage :topics :language :updated_at])) + (select-keys user-repo [:html_url :name :description :homepage :topics :language :stargazers_count :updated_at])) (defn repos-slim [user-repos] @@ -180,6 +188,7 @@ {:name (get-in first-repo [:owner :login]) :owner_url (get-in first-repo [:owner :html_url]) :languages (user-languages cleaned-repos) + :total-stars (reduce + (map :stargazers_count cleaned-repos)) :repositories cleaned-repos})) (defn recursive-curl diff --git a/test/git_hire/test_main.clj b/test/git_hire/test_main.clj index c88aa46..021ebb1 100644 --- a/test/git_hire/test_main.clj +++ b/test/git_hire/test_main.clj @@ -1,4 +1,7 @@ -(ns git-hire.test-main) +(ns git-hire.test-main + (:require + [clojure.test :as t])) + (require '[clojure.test :as t] '[babashka.classpath :as cp] '[git-hire.main :as main]) @@ -46,6 +49,7 @@ :homepage "www.foo.bar" :topics ["foo" "bar"] :language "clojure" + :stargazers_count 10 :updated_at "2020-01-01T00:00:00Z"}] (main/repos-slim [{:name "foo" :html_url "bar" @@ -68,6 +72,7 @@ :homepage "www.foo.bar" :topics ["foo" "bar"] :language "clojure" + :stargazers_count 10 :updated_at "2020-01-01T00:00:00Z"} (main/repo-slim {:name "foo" :html_url "bar" @@ -115,18 +120,18 @@ :pizza "turtles"}]))))) (t/deftest user-location-search-params-location - (t/is (= {:query-params {"per_page" 10, "q" "location:\"bad+kissingen\" repos:>=0"}} - (main/user-location-search-params-location 10 0 "Bad Kissingen"))) + (t/is (= {:query-params {"per_page" 10, "q" "location:\"bad+kreuznach\" repos:>=0"}} + (main/user-location-search-params-location 10 0 "Bad Kreuznach"))) (t/is (= {:query-params {"per_page" 20, "q" "location:\"mainz\" repos:>=0"}} (main/user-location-search-params-location 20 0 "Mainz")))) (t/deftest file-path-location-all (t/is (= "./profiles/mainz/all/" (main/file-path-location-all "Mainz"))) - (t/is (= "./profiles/bad kissingen/all/" - (main/file-path-location-all "Bad Kissingen")))) + (t/is (= "./profiles/bad kreuznach/all/" + (main/file-path-location-all "Bad Kreuznach")))) (t/deftest user-location-search-params-location-lang (t/is (= {:query-params {"per_page" 10, - "q" "location:\"bad+kissingen\" repos:>=0 language:\"clojure\""}} - (main/user-location-search-params-location-lang 10 0 "Bad Kissingen" "clojure")))) \ No newline at end of file + "q" "location:\"bad+kreuznach\" repos:>=0 language:\"clojure\""}} + (main/user-location-search-params-location-lang 10 0 "Bad Kreuznach" "clojure")))) \ No newline at end of file