46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
import re
|
|
import requests
|
|
import urllib.parse
|
|
from lxml import html
|
|
|
|
# Get web page with Russian cities from RP5
|
|
russian_cities_page = requests.get('https://rp5.ru/Погода_в_России')
|
|
russian_cities_tree = html.fromstring(russian_cities_page.content)
|
|
|
|
# Extract the table with cities
|
|
columns = russian_cities_tree.xpath('//div[@class="countryMap"]//div[@class="countryMap-cell"]')
|
|
|
|
# File
|
|
cities_file = open('russia.csv', 'wt')
|
|
cities_file.write('ID,City\n')
|
|
|
|
# Write all identifiers and cities
|
|
city_link_regex = re.compile(r'^https*://rp5.ru/(Погода_[А-Яа-я\w.,\-()]+)')
|
|
added_ids = []
|
|
for col in columns:
|
|
ids = col.xpath('.//span[@class="Ajax-PointID"]//@id')
|
|
for city in ids:
|
|
|
|
str_to_write = ''
|
|
# Check identifier
|
|
for added in added_ids:
|
|
if city == added:
|
|
break
|
|
else:
|
|
# Write identifier
|
|
added_ids.append(city)
|
|
str_to_write += str(city)
|
|
|
|
# Comment this block to execute script faster and prevent DDoS detecting on RP5 server
|
|
city_response = requests.get('https://rp5.ru/town.php?id=' + city)
|
|
redirect = urllib.parse.unquote(city_response.url)
|
|
print(redirect)
|
|
str_to_write += (',' + city_link_regex.search(redirect).group(1))
|
|
|
|
# Write to file
|
|
cities_file.write(str_to_write + '\n')
|
|
print(city)
|
|
|
|
cities_file.close()
|
|
print('')
|
|
input('Press ENTER to exit ...')
|