aaaaaaaaaaaaaaaaaaaaaaa

src/webScraper/DbEntity.py

@@ -52,7 +52,7 @@ class DbEntity:
                 print("Creating sqllite database")
                 # with open(os.path.join(self.currentFolder, "Spritpreise.db"), "w"): pass
 
-            connection, cursor = self.__getSqlLiteConnection()
+            connection, cursor = self._getSqlLiteConnection()
 
             cursor.execute(
                 "CREATE TABLE IF NOT EXISTS \"spritpreis_header\" (\
@@ -95,7 +95,7 @@ class DbEntity:
             cursor.close()
             connection.close()
 
-            connection, cursor = self.__getDbConnection()
+            connection, cursor = self._getDbConnection()
 
             cursor.execute(
                 "CREATE SEQUENCE IF NOT EXISTS spritpreise_header_id_seq;\

src/webScraper/Spritpreise.py

@@ -72,6 +72,7 @@ class Spritpreise(DbEntity):
 
     def getPrices(self, fuelType):
         page = 1
+        errorCount = 0
         while True:
             try:
                 print(f"fuelType: {fuelType} page: {page}")
@@ -88,7 +89,7 @@ class Spritpreise(DbEntity):
                     if entryName not in self.fuelInfos:
                         self.fuelInfos[entryName] = {"3": None, "5": None, "6": None, "7": None}
 
-                    city = re.match("(\d{5}) (.*?)", location["city"])
+                    city = re.match(r"(\d{5}) (.*?)", location["city"])
                     address = re.match(r"(.+?)\s+(\d+[a-zA-Z]?)$", location["street"])
 
                     self.fuelInfos[entryName]["street"] = address.group(1)
@@ -100,6 +101,11 @@ class Spritpreise(DbEntity):
                     self.fuelInfos[entryName][str(self.convertType(fuelType))] = price if price.replace(".", "").isnumeric() else None
                 page += 1
             except Exception as e:
+                errorCount += 1
+                if errorCount > 10:
+                    self.__writeLog(f"To many errors occurred")
+                    return
+
                 self.__writeLog(f"Error occurred: {e}")
                 time.sleep(5)
 
@@ -184,46 +190,3 @@ class Spritpreise(DbEntity):
             self.fuelInfos = json.load(f)
         return self
 
-
-    def createDb(self):
-        connection, cursor = self.__getDbConnection()
-        sql = ('CREATE SEQUENCE IF NOT EXISTS spritpreise_header_id_seq;\
-                \
-                -- Table Definition\
-                CREATE TABLE IF NOT EXISTS "public"."spritpreis_header" (\
-                    "id" int4 NOT NULL DEFAULT nextval(\'spritpreise_header_id_seq\'::regclass),\
-                    "name" text NOT NULL,\
-                    "street" text NOT NULL,\
-                    "city" text NOT NULL,\
-                    PRIMARY KEY ("id")\
-                );\
-                -- Indices\
-                CREATE UNIQUE INDEX IF NOT EXISTS spritpreise_header_pkey ON public.spritpreis_header USING btree (id);\
-                CREATE INDEX IF NOT EXISTS idx ON public.spritpreis_header USING btree (city) INCLUDE (city, street) WITH (deduplicate_items=\'false\');\
-               \
-               CREATE SEQUENCE IF NOT EXISTS spritpreis_position_id_seq;\
-                \
-                -- Table Definition\
-                CREATE TABLE IF NOT EXISTS "public"."spritpreis_position" (\
-                    "id" int4 NOT NULL DEFAULT nextval(\'spritpreis_position_id_seq\'::regclass),\
-                    "of_spritpreis_header" int4 NOT NULL,\
-                    "time" timestamptz NOT NULL,\
-                    "diesel_price" float4,\
-                    "e10_price" float4,\
-                    "e5_price" float4,\
-                    "super_plus_price" float4,\
-                    CONSTRAINT "foreign_idx" FOREIGN KEY ("of_spritpreis_header") REFERENCES "public"."spritpreis_header"("id") ON DELETE CASCADE,\
-                    PRIMARY KEY ("id")\
-                );\
-                -- Indices\
-                CREATE INDEX IF NOT EXISTS "fki_F" ON public.spritpreis_position USING btree (of_spritpreis_header);\
-                CREATE INDEX IF NOT EXISTS idx_time ON public.spritpreis_position USING btree ("time") WITH (deduplicate_items=\'true\');\
-                CREATE INDEX IF NOT EXISTS idx_diesel ON public.spritpreis_position USING btree (diesel_price) INCLUDE (diesel_price) WITH (deduplicate_items=\'true\');\
-                CREATE INDEX IF NOT EXISTS idx_e10 ON public.spritpreis_position USING btree (e10_price) WITH (deduplicate_items=\'true\');\
-                CREATE INDEX IF NOT EXISTS idx_e5 ON public.spritpreis_position USING btree (e5_price) WITH (deduplicate_items=\'true\');\
-                CREATE INDEX IF NOT EXISTS idx_super_plus ON public.spritpreis_position USING btree (super_plus_price) WITH (deduplicate_items=\'true\');')
-
-        cursor.execute(sql)
-
-        cursor.close()
-        connection.close()