Skip to content

Commit

Permalink
feat: add scrape by tweet id (#16)
Browse files Browse the repository at this point in the history
* feat: add scrape by tweet id

* use valid twitter id on the ut
  • Loading branch information
alvin-reyes authored Jan 21, 2025
1 parent 1eab24e commit 3b8c987
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 0 deletions.
10 changes: 10 additions & 0 deletions internal/jobs/twitter.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,16 @@ func (ws *TwitterScraper) ExecuteJob(j types.Job) (types.JobResult, error) {
return types.JobResult{
Data: dat,
}, err

case "getbyid":
tweet, err := twitter.ScrapeTweetByID(ws.configuration.DataDir, args.Query)
if err != nil {
return types.JobResult{Error: err.Error()}, err
}
dat, err := json.Marshal(tweet)
return types.JobResult{
Data: dat,
}, err
}

// Do the web scraping here
Expand Down
17 changes: 17 additions & 0 deletions internal/jobs/twitter/tweets.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,20 @@ func ScrapeTweetsByQuery(baseDir string, query string, count int) ([]*TweetResul
}
return tweets, nil
}

func ScrapeTweetByID(baseDir string, tweetID string) (*twitterscraper.Tweet, error) {
scraper, account, err := getAuthenticatedScraper(baseDir)
if err != nil {
return nil, err
}

tweet, err := scraper.GetTweet(tweetID)
if err != nil {
if handleRateLimit(err, account) {
return nil, err
}
return nil, err
}

return tweet, nil
}
19 changes: 19 additions & 0 deletions internal/jobs/twitter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,4 +104,23 @@ var _ = Describe("Twitter Scraper", func() {

Expect(results[0].Username).ToNot(BeEmpty())
})

It("should get tweet by ID", func() {
res, err := twitterScraper.ExecuteJob(types.Job{
Type: TwitterScraperType,
Arguments: map[string]interface{}{
"type": "getbyid",
"query": "1881258110712492142",
},
})
Expect(err).NotTo(HaveOccurred())
Expect(res.Error).To(BeEmpty())

var tweet *twitterscraper.Tweet
res.Unmarshal(&tweet)
Expect(err).NotTo(HaveOccurred())
Expect(tweet).NotTo(BeNil())
Expect(tweet.ID).To(Equal("1234567890"))
Expect(tweet.Text).NotTo(BeEmpty())
})
})

0 comments on commit 3b8c987

Please sign in to comment.