Repository navigation
Expand file tree
/
Copy pathpdf_processor.py
More file actions
118 lines (98 loc) · 3.55 KB
/
Copy pathpdf_processor.py
File metadata and controls
118 lines (98 loc) · 3.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# -*- coding: utf-8 -*-
"""PDF processing: download, text extraction, and language detection."""
import os
import time
import requests
from bs4 import BeautifulSoup
import fitz
from langdetect import detect, LangDetectException
from config import (
RBI_NOTIFICATION_URL,
RBI_PDFS_DIR,
HEADERS,
WIKI_HEADERS,
)
def download_pdf(url, save_dir=RBI_PDFS_DIR, filename=None):
os.makedirs(save_dir, exist_ok=True)
if filename is None:
filename = f"rbi_{int(time.time())}.pdf"
path = os.path.join(save_dir, filename)
response = requests.get(url, headers=HEADERS, timeout=20)
response.raise_for_status()
with open(path, "wb") as f:
f.write(response.content)
return path
def extract_text_from_pdf(pdf_path):
doc = fitz.open(pdf_path)
text = ""
for page in doc:
text += page.get_text()
doc.close()
return text
def detect_language(text):
if len(text.strip()) < 50:
return "empty_or_scanned"
try:
return detect(text[:2000])
except LangDetectException:
return "unknown"
def extract_and_check_language(pdf_path):
text = extract_text_from_pdf(pdf_path)
lang = detect_language(text)
return text, lang
def get_rbi_notification_page():
response = requests.get(RBI_NOTIFICATION_URL, headers=HEADERS, timeout=15)
response.raise_for_status()
return response.text
def extract_pdf_links(html):
soup = BeautifulSoup(html, "lxml")
pdf_links = []
for a in soup.find_all("a", href=True):
href = a["href"]
if ".PDF" in href.upper() and "rdocs" in href:
pdf_links.append(href)
return list(set(pdf_links))
def download_pdfs(pdf_links, save_dir=RBI_PDFS_DIR, limit=30):
os.makedirs(save_dir, exist_ok=True)
downloaded = []
for i, url in enumerate(pdf_links[:limit]):
try:
response = requests.get(url, headers=HEADERS, timeout=20)
if response.status_code == 200 and len(response.content) > 1000:
filename = f"rbi_{i:03d}.pdf"
path = os.path.join(save_dir, filename)
with open(path, "wb") as f:
f.write(response.content)
downloaded.append({
"file": filename,
"url": url,
"size_kb": len(response.content) // 1024,
})
print(f"✓ {filename} ({len(response.content) // 1024} kb)")
time.sleep(1)
except Exception as e:
print(f"✗ Failed: {url[:60]}... — {e}")
return downloaded
def validate_pdfs(pdf_dir=RBI_PDFS_DIR):
results = []
for f in sorted(os.listdir(pdf_dir)):
if f.endswith(".pdf"):
path = os.path.join(pdf_dir, f)
try:
doc = fitz.open(path)
results.append({"file": f, "valid": True, "pages": doc.page_count})
doc.close()
except Exception as e:
results.append({"file": f, "valid": False, "error": str(e)})
return results
def batch_extract_text(pdf_dir=RBI_PDFS_DIR, out_dir=None):
out_dir = out_dir or os.path.join(os.path.dirname(pdf_dir), "processed")
os.makedirs(out_dir, exist_ok=True)
for f in os.listdir(pdf_dir):
if f.endswith(".pdf"):
pdf_path = os.path.join(pdf_dir, f)
text = extract_text_from_pdf(pdf_path)
txt_file = f.replace(".pdf", ".txt")
with open(os.path.join(out_dir, txt_file), "w", encoding="utf-8") as out:
out.write(text)
print(f"Saved: {txt_file}")