Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,7 +9,8 @@ | |
| https://github.com/halfak/Objective-Revision-Evaluation-Service/blob/master/ores/label_reverted.py | ||
| >>> from bad_words_detection_system import * | ||
| >>> edits = [Edit(1, {'one':1, 'two': 2}, False), Edit(2, {'three':3}, True), | ||
| ... Edit(3, {'one':5, 'four': 1}, False)] | ||
| >>> bot = Bot() | ||
| >>> bot.parse_edits(edits) | ||
| >>> bot.parse_bad_edits() | ||
|
|
@@ -25,18 +26,18 @@ | |
| import sys | ||
| import traceback | ||
| import json | ||
| import time | ||
| from importlib import import_module | ||
| from collections import OrderedDict | ||
| # TODO: User argparse | ||
|
|
||
| from revscoring.extractors import APIExtractor | ||
| from revscoring.datasources import diff | ||
This comment has been minimized.
Sorry, something went wrong. |
||
|
|
||
| from mw import api | ||
| from mw.lib import reverts | ||
|
|
||
| base_file_path = '/data/project/dexbot/pywikibot-core/something_' | ||
|
|
||
|
|
||
| class Edit(object): | ||
|
|
@@ -71,37 +72,39 @@ def __init__(self, words_cache=None, bad_words_cache=None, no_docs=None): | |
| self.cache = False | ||
|
|
||
| def initiate_cache(self, words_cache, bad_words_cache, no_docs): | ||
| with open(words_cache, 'r') as f: | ||
| self.words_db = json.loads(f.read()) | ||
| with open(bad_words_cache, 'r') as f: | ||
| self.bad_edits.added_words = json.loads(f.read()) | ||
| with open(no_docs, 'r') as f: | ||
| self.counter = int(f.read()) | ||
|
|
||
| def parse_edits(self, edits): | ||
| for edit in edits: | ||
| # Since edits can be gen and len doesn't mean there | ||
| self.counter += 1 | ||
| if edit.reverted: | ||
| for word in edit.added_words: | ||
| self.bad_edits.added_words[word] = \ | ||
| self.bad_edits.added_words.get(word, 0) + \ | ||
| edit.added_words[word] | ||
| self.bad_words_db[word] = ( | ||
| self.bad_words_db.get(word, 0) + 1) | ||
| self.bad_counter += 1 | ||
| continue | ||
| for word in edit.added_words: | ||
| self.words_db[word] = self.words_db.get(word, 0) + 1 | ||
|
|
||
| def parse_bad_edits(self, numbers_to_show=10): | ||
| self.possible_bad_words = {} | ||
| self.stop_words = {} | ||
| if not self.cache: | ||
| self.counter += 1 | ||
| for word in self.bad_edits.added_words: | ||
| if not self.cache: | ||
| self.words_db[word] = self.words_db.get(word, 0) + 1 | ||
| if 'sh' in word or 'ch' in word: | ||
| continue | ||
| self.possible_bad_words[word] = self.tf_idf(word) | ||
| self.stop_words[word] = self.idf(word) | ||
| if numbers_to_show: | ||
|
|
@@ -124,14 +127,15 @@ def show_results(self, numbers_to_show): | |
| for word in self.possible_bad_words: | ||
| if self.possible_bad_words[word] >= lim: | ||
| res[word] = self.possible_bad_words[word] | ||
| res = OrderedDict( | ||
| sorted(res.items(), key=lambda t: t[1], reverse=True)) | ||
| res_text = [] | ||
| for word in res: | ||
| res_text.append(word) | ||
| res_text.sort() | ||
| res_text = "#" + '\n#'.join(res_text) | ||
| self.bad_words_res_text = res_text | ||
| with open('%s_%s.txt' % (base_file_path, time.time()), 'w') as f: | ||
| f.write(res_text) | ||
|
|
||
| def show_results2(self, numbers_to_show): | ||
|
|
@@ -149,18 +153,18 @@ def show_results2(self, numbers_to_show): | |
| res_text.sort() | ||
| res_text = "#" + '\n#'.join(res_text) | ||
| self.stop_words_res_text = res_text | ||
| with open('%s2_%s.txt' % (base_file_path, time.time()), 'w') as f: | ||
| f.write(res_text) | ||
|
|
||
| def dump(self): | ||
| new_db = {} | ||
| for word in self.bad_edits.added_words: | ||
| new_db[word] = self.words_db[word] | ||
| with open('words_db.txt', 'w') as f: | ||
| f.write(json.dumps(new_db)) | ||
| with open('bad_edits_words.txt', 'w') as f: | ||
| f.write(json.dumps(self.bad_edits.added_words)) | ||
| with open('no_docs.txt', 'w') as f: | ||
| f.write(json.dumps(self.counter)) | ||
|
|
||
|
|
||
|
|
@@ -209,8 +213,8 @@ def handle_args(): | |
|
|
||
| def bot_gen(rev_pages, language, api_url): | ||
|
|
||
| session = api.Session(api_url) | ||
| extractor = APIExtractor(session, language=language) | ||
This comment has been minimized.
Sorry, something went wrong. 
halfak
Member
|
||
|
|
||
| for rev_id, page_id in rev_pages: | ||
| sys.stderr.write(".") | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
You should be able to
solve(wikitext.revision.diff.datasources.words_added, cache={revision_oriented.revision.text: ..., revision_oriented.revision.parent.text: ...})to get the same wordlist thatrevscoring/ORESworks with. It would be great if you could try that and point out any issues so we can repair them as appropriate for the whole system.