-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathIndexer.py
150 lines (124 loc) · 4.82 KB
/
Indexer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
import os
import re
from collections import defaultdict, Counter
import pickle
import time
from nltk.tokenize import word_tokenize
from nltk.stem.porter import PorterStemmer
direc = r"Z:\Assignment1\Programs\ExtractedText"
def getStopwords(stopwordsFile):
'''get stopwords from the stopwords file'''
f = open(stopwordsFile, 'r')
stopwords = [line.rstrip() for line in f]
sws = dict.fromkeys(stopwords)
f.close()
return sws
def traversethroughdocs(tf, porter, sw, postingslist):
'''Traverses through all the docs in the directory indexing them'''
docID = 1
for root, dirs, files in os.walk(direc):
for fname in files:
current_file = "%s%s%s" % (os.path.abspath(root), os.path.sep, fname)
indexOfExtension = fname.find('.')
fname = fname[:indexOfExtension]
print(fname)
tokensindoc, tf = tokenizedoc(tf, porter, current_file, sw, fname, docID)
postingslist = insertinpostingslist(tokensindoc, fname, postingslist)
docID = docID + 1
return postingslist, tf
def tokenizedoc(tf, porter, current_file, sw, fname, docID):
'''Returns all the cleaned tokens within the given doc'''
file = open(current_file, 'r', encoding="utf-8")
tokensindoc = []
lines = file.readlines()
for line in lines:
line = line.lower()
line = re.sub(r'[^a-z0-9 ]', ' ', line)
tokensinline = word_tokenize(line)
tokensindoc.extend(tokensinline)
tokensindoc, tf = cleantokensofeachdoc(tf, porter, tokensindoc, sw, fname, docID)
file.close()
return tokensindoc, tf
def cleantokensofeachdoc(tf, porter, tokensindoc, sw, fname, docID):
'''Cleans a list of tokens - stemming, removing repetition,
and adds the cleaned tokens to the tf table'''
#Stemming
tokensindoc = [porter.stem(token) for token in tokensindoc]
#Removing stop words
tokensindoc = [x for x in tokensindoc if x not in sw]
TFhelper = dict(Counter(tokensindoc))
#TFHelper is a dictionary of tokens and their counts, in that doc,
#basically its a single column in the tf matrix
tf = addTotf(tf, TFhelper, fname, docID)
#Removing repetition
tokensindoc = set(tokensindoc)
return tokensindoc, tf
def insertinpostingslist(tokensindoc, fname, postingslist):
"""
Inserts all the given tokens of each doc into the postings list,
in a suitable format, for later retrieval
"""
if any(postingslist):
for token, term in [(token, term) for token in tokensindoc for term in postingslist.keys()]:
if token == term:
postingslist[term].append(fname)
else:
for token in tokensindoc:
postingslist[token] = [fname]
return postingslist
def addTotf(tf, TFhelper, fname, docID):
'''Adds tokens of each doc to the tf table'''
newterm = 1
for token in TFhelper.keys():
for term in tf.keys():
if term == token:
newterm = 0
tf[term].append(TFhelper[token])
if newterm == 1:
tf[token] = [TFhelper[token]]
for term in tf.keys():
if len(tf[term]) != docID:
tf[term].append(0)
return tf
def createIDF(postingslist):
'''Creates the idf table from the postingslist'''
idf = defaultdict(int)
for term in postingslist.keys():
idf[term] = len(postingslist[term])
return idf
def writeToFile(filename, index):
'''Writes the given index to a file, as an object,
using the pickle module'''
with open(filename + '.txt', 'wb') as f:
pickle.dump(index, f)
print("Written to file " + filename + ".txt")
def writeToHumanReadableFile(filename, index):
"""
Writes the given index to a file,
in a suitable format for human reference,
unlike an object which cannot be read
"""
with open(filename + 'readable.txt', 'w') as f:
for k, v in index.items():
f.write(str(k) + ' >>> ' + str(v) + '\n\n')
f.close()
print("Written to readable file " + filename + "readable.txt")
def main():
'''main function, calls other functions'''
porter = PorterStemmer()
SW = getStopwords("stopwords.dat")
postingslist = defaultdict(list)
tf = defaultdict(list)
start = time.time()
postingslist, tf = traversethroughdocs(tf, porter, SW, postingslist)
IDF = createIDF(postingslist)
end = time.time()
writeToFile("postingslist", postingslist)
writeToHumanReadableFile("postingslist", postingslist)
writeToFile("idf", IDF)
writeToHumanReadableFile("idf", IDF)
writeToFile("tf", tf)
writeToHumanReadableFile("tf", tf)
print("running time : " + str(end - start))
if __name__ == '__main__':
main()