Spaces:
Sleeping
Sleeping
| import sys | |
| import os | |
| sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) | |
| try: | |
| import requests # type: ignore | |
| except Exception: | |
| # Fallback lightweight wrapper around urllib when `requests` isn't available. | |
| import urllib.request as _urllib_request | |
| class _RequestsFallback: | |
| class Response: | |
| def __init__(self, text: str, status_code: int): | |
| self.text = text | |
| self.status_code = status_code | |
| def raise_for_status(self): | |
| if not (200 <= self.status_code < 300): | |
| raise Exception(f"HTTP {self.status_code}") | |
| def get(self, url, headers=None, timeout=None): | |
| req = _urllib_request.Request(url, headers=headers or {}) | |
| with _urllib_request.urlopen(req, timeout=timeout) as resp: | |
| body = resp.read().decode("utf-8", errors="ignore") | |
| return _RequestsFallback.Response(body, resp.getcode()) | |
| requests = _RequestsFallback() | |
| try: | |
| from bs4 import BeautifulSoup # type: ignore | |
| except Exception: | |
| # Lightweight fallback when `bs4` isn't installed. Implements the | |
| # minimal interface used in this module: constructor, calling with a | |
| # list of tag names to remove, get_text, and find('title'). | |
| import re | |
| import html as _html | |
| class _FallbackElement: | |
| def __init__(self, parent, tag): | |
| self._parent = parent | |
| self._tag = tag | |
| def extract(self): | |
| # remove all occurrences of the tag and its contents | |
| pattern = re.compile(rf"<%s\b[^>]*>.*?</%s>" % (re.escape(self._tag), re.escape(self._tag)), re.DOTALL | re.IGNORECASE) | |
| self._parent._html = pattern.sub("", self._parent._html) | |
| class BeautifulSoup: | |
| def __init__(self, html, parser=None): | |
| self._html = html or "" | |
| def __call__(self, tag_list): | |
| # return a list of elements with an extract() method | |
| return [_FallbackElement(self, tag) for tag in tag_list] | |
| def get_text(self, separator=" ", strip=True): | |
| # remove all tags | |
| text = re.sub(r"<[^>]+>", "", self._html) | |
| text = _html.unescape(text) | |
| if strip: | |
| text = text.strip() | |
| if separator != " ": | |
| # normalize whitespace according to separator | |
| text = re.sub(r"\s+", separator, text) | |
| else: | |
| text = re.sub(r"\s+", " ", text) | |
| return text | |
| def find(self, name): | |
| if name.lower() == 'title': | |
| m = re.search(r"<title\b[^>]*>(.*?)</title>", self._html, re.IGNORECASE | re.DOTALL) | |
| if m: | |
| class _T: | |
| def __init__(self, t): | |
| self.text = t | |
| return _T(_html.unescape(m.group(1).strip())) | |
| return None | |
| import importlib | |
| try: | |
| _celery_mod = importlib.import_module("celery") | |
| Celery = getattr(_celery_mod, "Celery") | |
| except Exception: | |
| # Provide a lightweight fallback Celery-compatible stub so the module | |
| # can be imported and used in environments without celery installed. | |
| class Celery: | |
| def __init__(self, *args, **kwargs): | |
| self.conf = type("C", (), {})() | |
| def task(self, *args, **kwargs): | |
| def decorator(fn): | |
| return fn | |
| return decorator | |
| try: | |
| from dotenv import load_dotenv # type: ignore | |
| except Exception: | |
| # If python-dotenv isn't installed, provide a no-op fallback so | |
| # this module can still run (env vars can come from the environment). | |
| def load_dotenv(): | |
| return | |
| load_dotenv() | |
| REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") | |
| celery_app = Celery("vertex_tasks", broker=REDIS_URL, backend=REDIS_URL) | |
| celery_app.conf.update( | |
| task_serializer="json", | |
| result_serializer="json", | |
| accept_content=["json"], | |
| timezone="UTC", | |
| enable_utc=True, | |
| worker_concurrency=4 | |
| ) | |
| def crawl_and_index_url(self, url: str, profile: str = "public"): | |
| print(f"🚀 Worker picked up task for URL: {url} | Workspace: {profile}") | |
| from services.ingestion import VertexIndexer | |
| from main import nvidia_manager | |
| headers = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) VertexEngineCrawler/1.0" | |
| } | |
| try: | |
| response = requests.get(url, headers=headers, timeout=10) | |
| response.raise_for_status() | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # 1. Clean the HTML | |
| for element in soup(["script", "style", "nav", "footer", "header"]): | |
| element.extract() | |
| raw_text = soup.get_text(separator=" ", strip=True)[:6000] | |
| if len(raw_text) < 150: | |
| return {"status": "skipped", "reason": "Content body too short."} | |
| # 2. Extract a Title purely from HTML (Not AI) | |
| title_tag = soup.find('title') | |
| page_title = title_tag.text.strip() if title_tag else "Untitled Webpage" | |
| # 3. Ask AI ONLY to clean the text (No JSON formatting requests) | |
| system_prompt = ( | |
| "You are an automated data extraction engine. I will provide raw webpage text. " | |
| "Your ONLY job is to return the clean, readable, factual body text of the article. " | |
| "Remove all UI elements, navigation text, and copyright notices. " | |
| "Do NOT add any conversational text like 'Here is the text'. Output ONLY the cleaned content." | |
| ) | |
| cleaned_text = nvidia_manager.get_completion( | |
| model="meta/llama-3.1-8b-instruct", | |
| messages=[ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": f"Raw Content:\n{raw_text}"} | |
| ], | |
| temperature=0.1 | |
| ).strip() | |
| # 4. Safely construct the dictionary in Python (No JSON Decoding needed) | |
| if len(cleaned_text) < 50: | |
| print(f"🛑 AI rejected {url} due to low quality content.") | |
| return {"status": "rejected", "reason": "AI determined content was not useful."} | |
| indexer = VertexIndexer() | |
| indexer.index_source( | |
| title=page_title, | |
| raw_text=cleaned_text, | |
| source_url=url, | |
| profile=profile | |
| ) | |
| return {"status": "completed", "title": page_title, "profile": profile} | |
| except Exception as exc: | |
| print(f"Error handling crawl on {url}. Retrying... Exception: {str(exc)}") | |
| raise self.retry(exc=exc, countdown=10) |