Here is the code to show how to do Web Scraping in Python
import requests
from lxml import html
from bs4 import BeautifulSoup
import csv
import pandas as pd
from openpyxl import Workbook
from openpyxl.utils import get_column_letter
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.worksheet.table import Table, TableStyleInfo
from datetime import datetime
page_number = 1
Baseurl = 'https://www.vesselfinder.com'
urls = 'https://www.vesselfinder.com/vessels'
header = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/39.0.2171.95 Safari/537.36',
}
# Get HTML Content
r = requests.get(urls, headers=header)
soup = BeautifulSoup(r.content, 'html.parser')
page_links = []
table = soup.find('table', class_='results')
for row in table.tbody.find_all('tr'):
# Find all data for each column
columns = row.find('td')
link = columns.a['href']
# print(link)
page_links.append(link)
my_heads = ['Key', 'Value']
wb = Workbook()
ws = wb.active
for link in page_links:
print(Baseurl + link)
r_inner = requests.get(Baseurl + link, headers=header)
df_list = pd.read_html(r_inner.text) # this parses all the tables in webpages to a list
df = df_list[1]
print(df)
# df.head()
for r in dataframe_to_rows(df, header=True, index=False):
ws.append(r)
style = TableStyleInfo(name="TableStyleMedium9", showRowStripes=True)
table = Table(displayName="Vessel_Info",
ref="A1:" + get_column_letter(ws.max_column) + str(ws.max_row))
table.tableStyleInfo = style
ws.add_table(table)
wb.save('Vessel_Info_' + datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p") + '.xlsx')
print('Next Page link :', Baseurl + soup.find('a', href=True, class_='pagination-next')['href'])
while True:
pagination = Baseurl + soup.find('a', href=True, class_='pagination-next')['href']
if pagination:
page_number += 1
print('Next Page link :', Baseurl + soup.find('a', href=True, class_='pagination-next')['href'])
header = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/39.0.2171.95 Safari/537.36',
}
# Get HTML Content
r = requests.get(pagination, headers=header)
soup = BeautifulSoup(r.content, 'html.parser')
else:
break
"""
with pd.ExcelWriter('Vessel_Info.xlsx', mode='a', engine='openpyxl') as writer:
df.to_excel(writer, sheet_name='Sheet_name_1', header=my_heads)
items = soup.findAll("a", {"class": "pagination-next"})
"""
Comments
Post a Comment