Code_webcrawler.py

import math 
import sqlite3 
import time
import string
from porterstemmer import PorterStemmer
from collections import deque
from urllib.parse import urljoin, urlparse, urldefrag
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
from bs4 import BeautifulSoup


# the higher ID 
tokens = 0
documents = 0
terms = 0
stop_words_found = 0

DB_PATH = "webcrawler.db"

MAX_FRONTIER_SIZE = 500

SKIP_EXTENSIONS = (
    ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".webp",
    ".svg", ".css", ".js", ".ico", ".zip", ".rar",
    ".mp3", ".mp4", ".avi", ".mov", ".wmv"
)

USER_AGENT = "CS3308-WebCrawler/1.0"

STOP_WORDS = {
    "a", "about", "after", "again", "all", "am", "an", "and", "any", "are",
    "as", "at", "be", "because", "been", "before", "being", "between",
    "both", "but", "by", "can", "did", "do", "does", "doing", "down",
    "during", "each", "few", "for", "from", "had", "has", "have", "having",
    "he", "her", "here", "him", "his", "how", "i", "if", "in", "into",
    "is", "it", "its", "itself", "just", "me", "more", "most", "my",
    "no", "nor", "not", "now", "of", "off", "on", "once", "only", "or",
    "other", "our", "out", "over", "own", "same", "she", "should", "so",
    "some", "such", "than", "that", "the", "their", "them", "then",
    "there", "these", "they", "this", "those", "through", "to", "too",
    "under", "until", "up", "very", "was", "we", "were", "what", "when",
    "where", "which", "while", "who", "why", "with", "would", "you",
    "your"
}

stemmer = PorterStemmer()


def normalize_url(base_url, link):
    """
    Converts relative links to absolute URLs and removes fragments.
    """
    joined_url = urljoin(base_url, link)
    clean_url = urldefrag(joined_url).url
    parsed = urlparse(clean_url)

    if parsed.scheme not in {"http", "https"}:
        return None
    
    if not parsed.netloc:
        return None
    
    return clean_url

def should_skip_url(url):
    """
    Returns True if the URL should not be crawled.
    Avoids images, scripts, PDFs, videos, and other non-HTML content.
    """
    parsed = urlparse(url)
    path = parsed.path.lower()

    if parsed.scheme not in {"http", "https"}:
        return True

    if path.endswith(SKIP_EXTENSIONS):
        return True

    return False

def fetch_page(url):
    """
    Downloads a web page and returns its HTML as text.
    Non-HTML reponses will be ignored
    """
    try:
        request = Request(url, headers={"User-Agent": USER_AGENT})

        with urlopen(request, timeout=10) as response:
            content_type = response.headers.get("Content-Type", "").lower()

            if "text/html" not in content_type:
                return None
            
            charset = response.headers.get_content_charset() or "utf-8"
            raw_html = response.read()

        return raw_html.decode(charset, errors="ignore")
    
    except (HTTPError, URLError, TimeoutError, ValueError):
        return None
    
def extract_text(html):
    """
    Removes HTML tags and returns clean visible page text.
    """
    soup = BeautifulSoup(html, "html.parser")

    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()

    text = soup.get_text(separator=" ")
    text = " ".join(text.split())

    return text

def extract_links(html, base_url):
    """
    Extracts and normalizes all links from the given HTML content.
    """
    soup = BeautifulSoup(html, "html.parser")
    links = []

    for tag in soup.find_all("a", href=True):
        normalized = normalize_url(base_url, tag["href"])
        if normalized is None:
            continue
        if should_skip_url(normalized):
            continue
        links.append(normalized)

    return links

#
# We will create a term object for each unique instance of a term
#
class Term():
    termid = 0
    termfreq = 0
    docs = 0 
    docids = {}

# split on any chars 
def splitchars(line) :
    return line.split()

def clean_token(token):
    # This applies the editing rules for Unit 4.
    # Returns the cleaned/stemmed token, or None is the token is not indexed

    global stop_words_found

    token = token.replace("\n", "").strip().lower()

    if token == "":
        return None
    # Remove terms beginning with punctuation
    # This will happen before stripping punctuation
    if token[0] in string.punctuation:
        return None
    # Remove trailing punctuation from otherwise valid tokens
    # Example: "running." becomes "running", but "(running" is rejected
    token = token.rstrip(string.punctuation)
    if token == "":
        return None
    # Remove terms that are numbers
    if token.isnumeric():
        return None
    # Remove stop words and count them
    if token in STOP_WORDS:
        stop_words_found += 1
        return None
    # Remove terms that are 2 characters or less
    if len(token) <= 2:
        return None
    # Stem remaining valid tokens
    token = stemmer.stem(token, 0, len(token) - 1)

    return token

# process the tokens of the source code 
def parsetoken(db, line):
    global documents 
    global tokens 
    global terms

    # this replaces any tab characters with a space character in the line
    # read from the file
    line = line.replace('\t',' ') 
    line = line.strip()
    
    #
    # This routine splits the contents of the line into tokens 
    l = splitchars(line)
    
    # for each token in the line process 
    for elmt in l:
        # Count every raw token parsed from the corpus
        tokens += 1

        # Apply our new editing rules for Unit 4
        cleaned_term = clean_token(elmt)

        # If clean_token returns None, this token is not indexed
        if cleaned_term is None:
            continue

        # If cleaned/stemmed term does not exist, add it
        if cleaned_term not in db:
            terms += 1
            db[cleaned_term] = Term()
            db[cleaned_term].termid = terms
            db[cleaned_term].docids = dict()
            db[cleaned_term].docs = 0

        # If this term is not in the document yet, add this to the term's postings list
        if documents not in db[cleaned_term].docids.keys():
            db[cleaned_term].docs += 1
            db[cleaned_term].docids[documents] = 0

        # Increment term frequency for this term in the document
        db[cleaned_term].docids[documents] += 1

    return l


def crawl_site(start_url, db, cur):
    """
    Crawls web pages starting from start_url, extracts text,
    sends page text to the tokenizer/indexer, and stores
    crawled URLs in DocumentDictionary.
    """
    global documents
    frontier = deque([start_url])
    queued_urls = set([start_url])
    crawled_urls = set()

    start_domain = urlparse(start_url).netloc

    while frontier:
        current_url = frontier.popleft()
        if current_url in crawled_urls:
            continue
        if should_skip_url(current_url):
            crawled_urls.add(current_url)
            continue
        
        print(f"Queue size: {len(frontier)} | Crawling: {current_url}")

        html = fetch_page(current_url)
        crawled_urls.add(current_url)

        if html is None:
            continue

        page_text = extract_text(html)

        if len(page_text.strip()) == 0:
            continue

        # Assign a new document ID before parsing.
        # This is because parsetoken() uses current document ID.
        documents += 1
        doc_id = documents

        # Store crawled URL as the document name.
        cur.execute(
            "insert into DocumentDictionary values (?, ?)",
            (current_url, doc_id)
        )

        # Send extracted page text into existing indexer logic.
        parsetoken(db, page_text)

        # Extract links from the current page and add them to frontier.
        links = extract_links(html, current_url)

        for link in links:
            if len(queued_urls) >= MAX_FRONTIER_SIZE:
                break

            parsed_link = urlparse(link)

            # Keep the crawl limited to the starting domain.
            # This avoids the crawler spreading uncontrollably across the web.
            if parsed_link.netloc != start_domain:
                continue

            if link not in queued_urls and link not in crawled_urls:
                frontier.append(link)
                queued_urls.add(link)

    return crawled_urls

def write_index_to_database(db, cur):
    for term in sorted(db.keys()):
        term_id = db[term].termid
        doc_freq = db[term].docs

        if doc_freq == 0:
            continue

        idf = math.log(documents / doc_freq)

        cur.execute(
            "insert into TermDictionary values (?, ?)",
            (term, term_id)
        )

        for doc_id, term_freq in db[term].docids.items():
            tfidf = term_freq * idf

            cur.execute(
                "insert into Posting values (?, ?, ?, ?, ?)",
                (term_id, doc_id, tfidf, doc_freq, term_freq)
            )

def get_cursor():
    conn = sqlite3.connect("indexer.db")
    return conn.cursor()
def select_all_records_by_author(cursor):
    sql = "SELECT * FROM DocumentDictionary"
    cursor.execute(sql)
    print(cursor.fetchall())  # or use fetchone()
    print("\nHere is a listing of the rows in the table DocumentDictionary\n")
    for row in cursor.execute("SELECT rowid, * FROM DocumentDictionary"):
        print(row)


"""
==========================================================================================
>>> main
==========================================================================================
"""
if __name__ == '__main__':

    # In memory index dictionary
    db = {}

    # Capture start time
    t2 = time.localtime()
    print ("Start Time: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))

    # Prompt for starting URL
    start_url = input("Enter URL to crawl (must be in the form http://www.domain.com): ").strip()

    # Create/connect to the SQLite database
    con = sqlite3.connect(DB_PATH) #This create database in the folder where your code file is saved.
    con.isolation_level = None
    cur = con.cursor()
    #
    # Create the inverted index tables.
    #
    # Document Dictionary Table
    cur.execute("drop table if exists DocumentDictionary") 
    cur.execute("drop index if exists idxDocumentDictionary")
    cur.execute("create table if not exists DocumentDictionary (DocumentName text, DocId int)") 
    cur.execute("create index if not exists idxDocumentDictionary on DocumentDictionary (DocId)")

    # Term Dictionary Table
    cur.execute("drop table if exists TermDictionary") 
    cur.execute("drop index if exists idxTermDictionary")
    cur.execute("create table if not exists TermDictionary (Term text, TermId int)") 
    cur.execute("create index if not exists idxTermDictionary on TermDictionary (TermId)")
 
    # Postings Table
    cur.execute("drop table if exists Posting") 
    cur.execute("drop index if exists idxPosting1") 
    cur.execute("drop index if exists idxPosting2")
    cur.execute("create table if not exists Posting (TermId int, DocId int, tfidf real, docfreq int, termfreq int)") 
    cur.execute("create index if not exists idxPosting1 on Posting (TermId)")
    cur.execute("create index if not exists idxPosting2 on Posting (Docid)")
    #
    # Crawl the website and build the in-memory index
    #
    crawl_site(start_url, db, cur)

    t2 = time.localtime()
    print("Indexing Complete, write to disk: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))

    #
    # Output files for inspection.
    #

    with open("Web_TermDictionary.txt", "w", encoding="utf-8") as term_file:
        for term in sorted(db.keys()):
            term_file.write(f"{term}\t{db[term].termid}\n")

    with open("Web_DocumentDictionary.txt", "w", encoding="utf-8") as docs_output:
        cur.execute("SELECT DocId, DocumentName FROM DocumentDictionary ORDER BY DocId")
        for docid, docname in cur.fetchall():
            docs_output.write(f"{docid}\t{docname}\n")

    #
    # Write the final inverted index to SQLite.
    #

    write_index_to_database(db, cur)

    con.commit()
    con.close()

    t2 = time.localtime()
    print("Database write complete: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))

    #
    # Print processing statistics
    #
 
    print("Website indexed: %s" % start_url)
    print("Number of documents processed %i" % documents)
    print("Total number of terms parsed from all documents %i" % tokens)
    print("Total number of unique terms found and added to the index %i" % terms)
    print("Total number of terms found that matched stop words %i" % stop_words_found)

    t2 = time.localtime()
    print("End Time: %.2d:%.2d" % (t2.tm_hour, t2.tm_min))