Term frequency (monolithic)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import string
import sys

stopfile = sys.argv[1]
txtfile = sys.argv[2]
word_freqs = []

with open(stopfile) as sw:
    stop_words = sw.read().split(',')
stop_words.extend(list(string.ascii_lowercase))

# This isn't fast, so we need a progress indicator.
linecount = 0
print(f'Processing {txtfile}...')

# Process the file one line at a time.
for line in open(txtfile):

    linecount += 1
    print(f'{linecount} lines     ', end='\r')
    sys.stdout.flush()

    # This will store the index position of the first character in a word.
    start_idx = None

    # We'll eventually increment this way down below where we can't
    # see it from here.
    char_idx = 0

    # Process the line character by character.
    for c in line:

        # If we're clean...
        if not start_idx:

            # Determine if we're at the beginning of a word.
            # If we are, set `start_char`;
            if c.isalnum():
                start_idx = char_idx
        else:

            # Determine if we're at the end of a word.
            if not c.isalnum():
                found = False

                # Get the word and reformat it.
                word = line[start_idx:char_idx].lower()

                # If this word isn't a stop word, we will tabulate
                # it. Use `wf_idx` to store the index of this word
                # in `word_freqs`.
                if word not in stop_words:
                    wf_idx = 0

                    # Iterate over `wf_idx` looking for this word. If it 
                    # is already in `word_freqs`, increment its count, set
                    # `found` to `True`, and break. Otherwise increment
                    # `wf_idx` and continue.
                    for record in word_freqs:
                        if word == record[0]:
                            record[1] += 1
                            found = True
                            break
                        wf_idx += 1

                    # If this word is not already in `word_freqs`,
                    # append it.
                    if not found:
                        word_freqs.append([word, 1])

                    # If `word_freqs` already has 2 or more entries,
                    # keep it ordered.
                    elif len(word_freqs) > 1:
                        for n in reversed(range(wf_idx)):
                            if word_freqs[wf_idx][1] > word_freqs[n][1]:
                                word_freqs[n], word_freqs[wf_idx] = \
                                    word_freqs[wf_idx], word_freqs[n]
                                wf_idx = n
                
                # We're done with this word. Reset the value of the
                # variable we've been using to preserve the index
                # of its first character.
                start_idx = None
        
        # Prepare to proceed to the next character in the line.
        char_idx += 1

# Display the 25 records with the greatest frequency values.
print()
for tf in word_freqs[0:25]:
    print(f'{tf[0]}: {tf[1]}')