slqite fix
This commit is contained in:
@@ -1,21 +1,22 @@
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pprint import pprint
|
||||
from typing import Literal
|
||||
import os
|
||||
import mysql.connector
|
||||
import bs4
|
||||
import pytz
|
||||
import requests
|
||||
import psycopg2
|
||||
|
||||
import time
|
||||
import threading
|
||||
import sched
|
||||
|
||||
from mysql.connector.aio.charsets import charsets
|
||||
from src.webScraper.DbEntity import DbEntity
|
||||
|
||||
|
||||
class Spritpreise:
|
||||
class Spritpreise(DbEntity):
|
||||
def __init__(self, location:str, radius:int, fuelType:Literal["diesel", "E5", "E10", "super plus"] = "E10"):
|
||||
self.location = location
|
||||
self.radius = radius
|
||||
@@ -28,40 +29,14 @@ class Spritpreise:
|
||||
"super E5": 7,
|
||||
}
|
||||
self.fuelInfos = {}
|
||||
self.currentFolder = os.path.dirname(os.path.realpath(__file__))
|
||||
self.dbType = ""
|
||||
self.dbName = ""
|
||||
self.dbHost = ""
|
||||
self.dbPort = ""
|
||||
self.dbUser = ""
|
||||
self.dbPassword = ""
|
||||
super().__init__()
|
||||
|
||||
|
||||
|
||||
def getCurrentTime(self):
|
||||
return datetime.now().strftime("%Y/%m/%d %H:%M:%S")
|
||||
|
||||
|
||||
def setDbConnection(self, dbHost:str, dbPort:str, dbUser, dbPassword:str, dbType:Literal["mysql", "pgsql"]):
|
||||
self.dbType = dbType
|
||||
self.dbName = "Spritpreise"
|
||||
self.dbHost = dbHost
|
||||
self.dbPort = dbPort
|
||||
self.dbUser = dbUser
|
||||
self.dbPassword = dbPassword
|
||||
|
||||
def __getDbConnection(self):
|
||||
if self.dbType == "mysql":
|
||||
connection = mysql.connector.connect(host=self.dbHost, user=self.dbUser, password=self.dbPassword,
|
||||
database=self.dbName, port=self.dbPort)
|
||||
elif self.dbType == "pgsql":
|
||||
connection = psycopg2.connect(dbname=self.dbName, user=self.dbUser, password=self.dbPassword,
|
||||
host=self.dbHost, port=self.dbPort)
|
||||
|
||||
cursor = connection.cursor()
|
||||
|
||||
return connection, cursor
|
||||
|
||||
|
||||
def __writeLog(self, text: str, printOnConsole=True):
|
||||
now = ""
|
||||
try:
|
||||
@@ -74,6 +49,7 @@ class Spritpreise:
|
||||
if printOnConsole:
|
||||
print(f"[{now}] {text}\n")
|
||||
|
||||
|
||||
def convertType(self, fuelType:str):
|
||||
if isinstance(fuelType, int):
|
||||
return fuelType
|
||||
@@ -111,8 +87,14 @@ class Spritpreise:
|
||||
entryName = f'{location["city"]} {location["street"]}'
|
||||
if entryName not in self.fuelInfos:
|
||||
self.fuelInfos[entryName] = {"3": None, "5": None, "6": None, "7": None}
|
||||
self.fuelInfos[entryName]["street"] = location["street"]
|
||||
self.fuelInfos[entryName]["city"] = location["city"]
|
||||
|
||||
city = re.match("(\d{5}) (.*?)", location["city"])
|
||||
address = re.match(r"(.+?)\s+(\d+[a-zA-Z]?)$", location["street"])
|
||||
|
||||
self.fuelInfos[entryName]["street"] = address.group(1)
|
||||
self.fuelInfos[entryName]["houseNumber"] = address.group(2)
|
||||
self.fuelInfos[entryName]["zipCode"] = city.group(1)
|
||||
self.fuelInfos[entryName]["city"] = city.group(2)
|
||||
self.fuelInfos[entryName]["name"] = location["name"]
|
||||
self.fuelInfos[entryName]["time"] = datetime.now(pytz.timezone('Europe/Berlin')).strftime('%Y-%m-%d %H:%M:%S%z')[:-2]
|
||||
self.fuelInfos[entryName][str(self.convertType(fuelType))] = price if price.replace(".", "").isnumeric() else None
|
||||
@@ -121,6 +103,7 @@ class Spritpreise:
|
||||
self.__writeLog(f"Error occurred: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def getAllPrices(self):
|
||||
start = time.time()
|
||||
"""
|
||||
@@ -163,24 +146,22 @@ class Spritpreise:
|
||||
time.sleep(sleepTime)
|
||||
|
||||
|
||||
|
||||
def insertIntoDb(self):
|
||||
connection, cursor = self.__getDbConnection()
|
||||
connection, cursor = self._getSqlLiteConnection()
|
||||
|
||||
for key, value in self.fuelInfos.items():
|
||||
cursor.execute('SELECT id FROM spritpreis_header WHERE city = %s and street = %s', (value['city'], value["street"]))
|
||||
|
||||
cursor.execute('SELECT id FROM spritpreis_header WHERE city = ? and street = ?', (value['city'], value["street"]))
|
||||
result = cursor.fetchone()
|
||||
|
||||
if result is None:
|
||||
cursor.execute("INSERT INTO spritpreis_header (city, street, name) VALUES (%s, %s, %s)", (value['city'], value["street"], value["name"]))
|
||||
cursor.execute('SELECT id FROM spritpreis_header WHERE city = %s and street = %s', (value['city'], value["street"]))
|
||||
cursor.execute("INSERT INTO spritpreis_header (city, street, name, houseNumber, zipCode) VALUES (?, ?, ?, ?, ?)", (value['city'], value["street"], value["name"], value["houseNumber"], value["zipCode"]))
|
||||
cursor.execute('SELECT id FROM spritpreis_header WHERE city = ? and street = ?', (value['city'], value["street"]))
|
||||
result = cursor.fetchone()
|
||||
print(result)
|
||||
|
||||
id = result[0]
|
||||
|
||||
cursor.execute("INSERT INTO spritpreis_position (of_spritpreis_header, time, diesel_price, e10_price, e5_price, super_plus_price) VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(id, value["time"], value["3"], value["5"], value["7"], value["6"]))
|
||||
cursor.execute("INSERT INTO spritpreis_position (of_spritpreis_header, time, diesel_price, e10_price, e5_price, super_plus_price) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(result[0], value["time"], value["3"], value["5"], value["7"], value["6"]))
|
||||
|
||||
cursor.close()
|
||||
connection.commit()
|
||||
@@ -189,7 +170,7 @@ class Spritpreise:
|
||||
return self
|
||||
|
||||
|
||||
def exportAsJson(self, outputFile):
|
||||
def exportAsJson(self, outputFile:str):
|
||||
start = time.time()
|
||||
with open(outputFile, "w", encoding="utf-8") as f:
|
||||
json.dump(self.fuelInfos, f, indent=4)
|
||||
@@ -198,7 +179,7 @@ class Spritpreise:
|
||||
|
||||
return self
|
||||
|
||||
def getDictFromJson(self, inputFile):
|
||||
def getDictFromJson(self, inputFile:str):
|
||||
with open(inputFile, "r", encoding="utf-8") as f:
|
||||
self.fuelInfos = json.load(f)
|
||||
return self
|
||||
|
||||
Reference in New Issue
Block a user