Code_indexer.py

import sys,os,re
import math 
import sqlite3 
import time
import string
from porterstemmer import PorterStemmer

# the database is a simple dictionnary 
database = {}

# regular expression for: extract words, extract ID from path, check for hexa value 
chars = re.compile(r'\W+')
pattid= re.compile(r'(\d{3})/(\d{3})/(\d{3})')

# the higher ID 
tokens = 0
documents = 0
terms = 0
stop_words_found = 0

STOP_WORDS = {
    "a", "about", "after", "again", "all", "am", "an", "and", "any", "are",
    "as", "at", "be", "because", "been", "before", "being", "between",
    "both", "but", "by", "can", "did", "do", "does", "doing", "down",
    "during", "each", "few", "for", "from", "had", "has", "have", "having",
    "he", "her", "here", "him", "his", "how", "i", "if", "in", "into",
    "is", "it", "its", "itself", "just", "me", "more", "most", "my",
    "no", "nor", "not", "now", "of", "off", "on", "once", "only", "or",
    "other", "our", "out", "over", "own", "same", "she", "should", "so",
    "some", "such", "than", "that", "the", "their", "them", "then",
    "there", "these", "they", "this", "those", "through", "to", "too",
    "under", "until", "up", "very", "was", "we", "were", "what", "when",
    "where", "which", "while", "who", "why", "with", "would", "you",
    "your"
}

stemmer = PorterStemmer()

#
# We will create a term object for each unique instance of a term
#
class Term():
    termid = 0
    termfreq = 0
    docs = 0 
    docids = {}

# split on any chars 
def splitchars(line) :
    return line.split()

def clean_token(token):
    # Returns the cleaned/stemmed token, or None is the token is not indexed

    global stop_words_found

    token = token.replace("/n", "").strip().lower()

    if token == "":
        return None
    # Remove terms beginning with punctuation
    # This will happen before stripping punctuation
    if token[0] in string.punctuation:
        return None
    # Remove trailing punctuation from otherwise valid tokens
    # Example: "running." becomes "running", but "(running" is rejected
    token = token.rstrip(string.punctuation)
    if token == "":
        return None
    # Remove terms that are numbers
    if token.isnumeric():
        return None
    # Remove stop words and count them
    if token in STOP_WORDS:
        stop_words_found += 1
        return None
    # Remove terms that are 2 characters or less
    if len(token) <= 2:
        return None
    # Stem remaining valid tokens
    token = stemmer.stem(token, 0, len(token) - 1)

    return token

# process the tokens of the source code 
def parsetoken(line):
    global documents 
    global tokens 
    global terms

    # this replaces any tab characters with a space character in the line
    # read from the file
    line = line.replace('\t',' ') 
    line = line.strip()
    
    #
    # This routine splits the contents of the line into tokens 
    l = splitchars(line)
    
    # for each token in the line process 
    for elmt in l:
        # Count every raw token parsed from the corpus
        tokens += 1

        # Apply our new editing rules for Unit 4
        cleaned_term = clean_token(elmt)

        # If clean_token returns None, this token is not indexed
        if cleaned_term is None:
            continue

        # If cleaned/stemmed term does not exist, add it
        if cleaned_term not in database:
            terms += 1
            database[cleaned_term] = Term()
            database[cleaned_term].termid = terms
            database[cleaned_term].docids = dict()
            database[cleaned_term].docs = 0

        # If this term is not in the document yet, add this to the term's postings list
        if documents not in database[cleaned_term].docids.keys():
            database[cleaned_term].docs += 1
            database[cleaned_term].docids[documents] = 0

        # Increment term frequency for this term in the document
        database[cleaned_term].docids[documents] += 1
    
    return l

#
# Open and read the file line by line, parsing for tokens and processing. All of the tokenizing
# is done in the parsetoken() function.
#
def process(filename): 
    try:
        file = open(filename, 'r') 
    except IOError:
        print("Error in file %s" % filename) 
        return False
    else:
        for l in file.readlines(): 
            parsetoken(l)
    file.close()

#
# This function will scan through the specified directory structure selecting
# every file for processing by the tokenizer function. 
# The walkdir function will also insert a row into the DocumentDictionary for each document processed.
#
def walkdir(cur, dirname): 
    global documents 
    all = {}
    all = [f for f in os.listdir(dirname) if os.path.isdir(os.path.join(dirname, f)) or os.path.isfile(os.path.join(dirname, f))] 
    for f in all:
        if os.path.isdir(dirname + '/' + f): 
            walkdir(cur, dirname + '/' + f)
        else:
            documents += 1
            cur.execute("insert into DocumentDictionary values (?, ?)", (dirname+'/'+f, documents)) 
            process(dirname + '/' + f)
             
    return True

def write_index_to_database(cur):
    for term in sorted(database.keys()):
        term_id = database[term].termid
        doc_freq = database[term].docs

        idf = math.log(documents / doc_freq)

        cur.execute(
            "insert into TermDictionary values (?, ?)",
            (term, term_id)
        )

        for doc_id, term_freq, in database[term].docids.items():
            tfidf = term_freq * idf

            cur.execute(
                "insert into Posting values (?, ?, ?, ?, ?)",
                (term_id, doc_id, tfidf, doc_freq, term_freq)
            )

def get_cursor():
    conn = sqlite3.connect("indexer.db")
    return conn.cursor()
def select_all_records_by_author(cursor):
    sql = "SELECT * FROM DocumentDictionary"
    cursor.execute(sql)
    print(cursor.fetchall())  # or use fetchone()
    print("\nHere is a listing of the rows in the table DocumentDictionary\n")
    for row in cursor.execute("SELECT rowid, * FROM DocumentDictionary"):
        print(row)


"""
==========================================================================================
>>> main
==========================================================================================

"""
if __name__ == '__main__':
    #
    # Print the start time of the program
    #
    t2 = time.localtime()
    print ("Start Time: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))
    #
    # The folder variable should be set to the path of the folder containing the corpus.
    # The walkdir function will recursively scan through the folder and all subfolders to find files to process.
    # The walkdir function will also insert a row into the DocumentDictionary for each document processed.
    
    # SPECIAL NOTE: Always place your code and corpus folder (cacm folder) in same folder
    folder = r"C:\Directory\Containing\Corpus"
    #
    # The following section will create the database and tables for the inverted index.
    # If the database or tables already exist, they will be dropped and recreated.
    #
    con = sqlite3.connect("indexer_part2.db") #This create database in the folder where your code file is saved.
    con.isolation_level = None
    cur = con.cursor()
    #
    # In the following section, three tables and their associated indexes will be created.
    # Before creating the table or index we will attempt to drop any existing tables in
    # case they exist
    #
    # Document Dictionary Table
    cur.execute("drop table if exists DocumentDictionary") 
    cur.execute("drop index if exists idxDocumentDictionary")
    cur.execute("create table if not exists DocumentDictionary (DocumentName text, DocId int)") 
    cur.execute("create index if not exists idxDocumentDictionary on DocumentDictionary (DocId)")
    # Term Dictionary Table
    cur.execute("drop table if exists TermDictionary") 
    cur.execute("drop index if exists idxTermDictionary")
    cur.execute("create table if not exists TermDictionary (Term text, TermId int)") 
    cur.execute("create index if not exists idxTermDictionary on TermDictionary (TermId)")
    # Postings Table
    cur.execute("drop table if exists Posting") 
    cur.execute("drop index if exists idxPosting1") 
    cur.execute("drop index if exists idxPosting2")
    cur.execute("create table if not exists Posting (TermId int, DocId int, tfidf real, docfreq int, termfreq int)") 
    cur.execute("create index if not exists idxPosting1 on Posting (TermId)")
    cur.execute("create index if not exists idxPosting2 on Posting (Docid)")
    #
    # The walkdir method essentially executes the indexer. The walkdir method will
    # read the corpus directory, Scan all files, parse tokens, and create the inverted index.
    #
    walkdir(cur, folder)

    with open("CACM_TermDictionary.txt", "w", encoding="utf-8") as term_file:
        for term in sorted(database.keys()):
            term_file.write(f"{term}\t{database[term].termid}\n")

    with open("CACM_DocumentDictionary.txt", "w", encoding="utf-8") as docs_output:
        cur.execute("SELECT DocId, DocumentName FROM DocumentDictionary ORDER BY DocId")
        for docid, docname in cur.fetchall():
            docs_output.write(f"{docid}\t{docname}\n")

    print("Indexing Complete, write to disk: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))

    write_index_to_database(cur)

    con.commit()
    con.close()

    t2 = time.localtime()
    print("Database write complete: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))

    #
    # Create the inverted index tables.
    
    #
    # Insert a row into the TermDictionary for each unique term along with a termid which is
    # a integer assigned to each term by incrementing an integer
    
    # Insert a row into the posting table for each unique combination of Docid and termid
    #
    # The following execute statement will show all the values inserted in TermDictionary table.
    # print("The content of TermDictionary table are as follows:")
    # cur.execute("select * from TermDictionary")
    # print(cur.fetchall())

    #
    # Print processing statistics
    #
 
    print ("Number of documents processed %i" % documents)
    print ("Total number of terms parsed from all documents %i" % tokens)
    print ("Total number of unique terms found and added to the index %i" % terms)
    print ("Total number of terms found that matched stop words %i" % stop_words_found)
    t2 = time.localtime()
    print ("End Time: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))