利用大语言模型增强网络抓取：一种现代化的方法

from bs4 import BeautifulSoupimport requestsdef extract_html_from_url(url):try:# Fetch HTML content from the URL using requestsresponse = requests.get(url)response.raise_for_status()# Raise an exception for bad responses (4xx and 5xx)# Parse HTML content using BeautifulSoupsoup = BeautifulSoup(response.content, "html.parser")excluded_tagNames = ["footer", "nav"]# Exclude elements with tag names 'footer' and 'nav'for tag_name in excluded_tagNames:for unwanted_tag in soup.find_all(tag_name):unwanted_tag.extract()# Process the soup to maintain hrefs in anchor tagsfor a_tag in soup.find_all("a"):href = a_tag.get("href")if href:a_tag.string = f"{a_tag.get_text()} ({href})"return ' '.join(soup.stripped_strings)# Return text content with preserved hrefsexcept requests.exceptions.RequestException as e:print(f"Error fetching data from {url}: {e}")return None

from pydantic import BaseModel, Fieldfrom typing import Optionalclass Activity(BaseModel):title: str = Field(description="The title of the activity.")rating: float = Field(description="The average user rating out of 10.")reviews_count: int = Field(description="The total number of reviews received.")travelers_count: Optional[int] = Field(description="The number of travelers who have participated.")cancellation_policy: Optional[str] = Field(description="The cancellation policy for the activity.")description: str = Field(description="A detailed description of what the activity entails.")duration: str = Field(description="The duration of the activity, usually given in hours or days.")language: Optional[str] = Field(description="The primary language in which the activity is conducted.")category: str = Field(description="The category of the activity, such as 'Boat Trip', 'City Tours', etc.")price: float = Field(description="The price of the activity.")currency: str = Field(description="The currency in which the price is denominated, such as USD, EUR, GBP, etc.")class ActivityScrapper(BaseModel):Activities: list[Activity] = Field("List of all the activities listed in the text")

from langchain.prompts import PromptTemplatefrom langchain.output_parsers import PydanticOutputParserfrom langchain_openai import ChatOpenAIfrom dotenv import load_dotenvload_dotenv()llm = ChatOpenAI(temperature=0)output_parser = PydanticOutputParser(pydantic_object = ActivityScrapper)prompt_template = """You are an expert making web scrapping and analyzing HTML raw code.If there is no explicit information don't make any assumption.Extract all objects that matched the instructions from the following html{html_text}Provide them in a list, also if there is a next page link remember to add it to the object.Please, follow carefulling the following instructions{format_instructions}"""prompt = PromptTemplate(template=prompt_template,input_variables=["html_text"],partial_variables={"format_instructions": output_parser.get_format_instructions})chain = prompt | llm | output_parser

url = "https://www.civitatis.com/es/budapest/"html_text_parsed = extract_html_from_url(url)activites = chain.invoke(input={"html_text": html_text_parsed})activites.Activities

[Activity(title='Paseo en barco al anochecer', rating=8.4, reviews_count=9439, travelers_count=118389, cancellation_policy='Cancelación gratuita', description='En este crucero disfrutaréis de las mejores vistas de Budapest cuando se viste de gala, al anochecer. El barco es panorámico y tiene partes descubiertas.', duration='1 hora', language='Español', category='Paseos en barco', price=21.0, currency='€'), Activity(title='Visita guiada por el Parlamento de Budapest', rating=8.8, reviews_count=2647, travelers_count=34872, cancellation_policy='Cancelación gratuita', description='El Parlamento de Budapest es uno de los edificios más bonitos de la capital húngara. Comprobadlo vosotros mismos en este tour en español que incluye la entrada.', duration='2 horas', language='Español', category='Visitas guiadas y free tours', price=27.0, currency='€') ...]