sqlite_integrated
1import pandas as pd 2import sqlite3 3import os 4from dataclasses import dataclass 5 6 7@dataclass 8class ForeignKey: 9 """Class representing an sql foreign key""" 10 11 table: str 12 """The table the foreign key points to""" 13 14 to_col: str 15 """Column the foreign key points to""" 16 17 from_col: str = None 18 """Column in current table, containing the key value""" 19 20 id: int = None 21 """The foreign key id""" 22 23 seq: int = None 24 """The foreign key sequence attribute""" 25 26 on_update: str = None 27 """The action the column will do if the data the key is pointing to changes. (Provide sql action).""" 28 29 on_delete: str = None 30 """The action the column will do if the data the key is pointing to changes. (Provide sql action).""" 31 32 match: str = None 33 34 def to_sql(self): 35 rep = f"FOREIGN KEY ({self.from_col}) REFERENCES {self.table} ({self.to_col})" 36 if self.on_update: 37 rep += f" ON UPDATE {self.on_update}" 38 if self.on_delete: 39 rep += f" ON DELETE {self.on_delete}" 40 return(rep) 41 42@dataclass 43class Column: 44 """Class representing en sql column.""" 45 46 def __init__(self, name: str, type: str, not_null: bool = None, default_value: any = None, primary_key: bool = False, col_id: int = None, foreign_key: ForeignKey = None) -> None: 47 48 if primary_key and type.upper() != "INTEGER": 49 raise DatabaseError(f"Primary key columns must have sqlite type: `INTEGER` not \'{type}\'") 50 51 self.name = name 52 """Name of the column.""" 53 54 self.type = type 55 """Type of the data in the column.""" 56 57 self.not_null = not_null 58 """Sql NOT NULL constraint.""" 59 60 self.default_value = default_value 61 """Sql DEFAULT. Default value for the column.""" 62 63 self.primary_key = primary_key 64 """Sql PRIMARY KEY. Automatic column that ensures that every entry has a unique.""" 65 66 self.col_id = col_id 67 """Id if the column in the table.""" 68 69 if foreign_key: 70 foreign_key.from_col = name 71 72 self.foreign_key = foreign_key 73 """ForeignKey object, that representing an sql foreign key.""" 74 75 76 def __repr__(self) -> str: 77 attrs = [] 78 if self.col_id: 79 attrs.append(str(self.col_id)) 80 attrs.append(self.name) 81 attrs.append(self.type) 82 if self.not_null: 83 attrs.append("NOT NULL") 84 if self.default_value: 85 attrs.append(f"DEFAULT: {self.default_value}") 86 if self.primary_key: 87 attrs.append("PRIMARY KEY") 88 if self.foreign_key: 89 attrs.append(self.foreign_key.to_sql()) 90 return(f"Column({', '.join(attrs)})") 91 92 93class DatabaseEntry(dict): 94 """ 95 A python dictionary that keeps track of the table it belongs to. This class is not meant to be created manually. 96 97 Parameters 98 ---------- 99 entry_dict : dict 100 A dictionary containing all the information. This information can be accesed just like any other python dict with `my_entry[my_key]`. 101 table : str 102 The name of the table the entry is a part of 103 """ 104 105 def __init__(self, entry_dict: dict, table: str): 106 self.table = table 107 self.update(entry_dict) 108 109 110 @classmethod 111 def from_raw_entry(cls, raw_entry: tuple, table_fields: list, table_name: str): 112 """ 113 Alternative constructor for converting a raw entry to a DatabaseEntry. 114 115 Parameters 116 ---------- 117 raw_entry : tuple 118 A tuple with the data for the entry. Ex: `(2, "Tom", "Builder", 33)` 119 table_fields : list 120 A list of column names for the data. Ex: `["id", "FirstName", "LastName", "Age"]` 121 table_name : str 122 The name of the table (in the database) that the data belongs to. Ex: "people" 123 """ 124 125 entry_dict = {} 126 127 if isinstance(table_fields, str): 128 table_fields = string_to_list(table_fields) 129 elif not isinstance(table_fields, list): 130 raise ValueError(f"table_fields must be either `list` or `str`. Got: {table_fields}") 131 132 if len(raw_entry) != len(table_fields): 133 raise DatabaseError(f"There must be as many names for table fields as there are fields in the entry: len({raw_entry}) != len({table_fields}) => {len(raw_entry)} != {len(table_fields)}") 134 135 for n, field in enumerate(table_fields): 136 entry_dict[field] = raw_entry[n] 137 entry = DatabaseEntry(entry_dict, table_name) 138 return(entry) 139 140 141 def __repr__(self) -> str: 142 """Represent a Database entry""" 143 144 return f"DatabaseEntry(table: {self.table}, data: {super().__repr__()})" 145 146 147def raw_table_to_table(raw_table: list, fields: list, table_name: str) -> list[DatabaseEntry]: 148 """ 149 Convert a raw table (list of tuples) to a table (generator of DatabaseEntry). 150 151 Parameters 152 ---------- 153 raw_table : list 154 A list of tuples with the data for the entries. 155 fields : list 156 A list of column names for the data. Ex: `["id", "FirstName", "LastName", "Age"]` 157 table_name: str 158 The name of the table (in the database) that the data belongs to. Ex: "people". 159 """ 160 161 if len(raw_table) == 0: 162 return 163 if len(raw_table[0]) != len(fields): 164 raise DatabaseError(f"There must be one raw column per field. {raw_table[0] = }, {fields = }") 165 166 for raw_entry in raw_table: 167 entry = {} 168 for n, field in enumerate(fields): 169 entry[field] = raw_entry[n] 170 yield DatabaseEntry(entry, table_name) 171 172 173def string_to_list(string: str) -> list: 174 """Takes a string with comma seperated values, returns a list of the values. (spaces are ignored)""" 175 176 return(string.replace(" ", "").split(",")) 177 178def value_to_sql_value(value) -> str: 179 """Converts python values to sql values. Basically just puts quotes around strings and not ints or floats. Also converts None to null""" 180 181 if isinstance(value, str): 182 return("'" + value.replace("'", "''") + "'") 183 elif isinstance(value, int): 184 return(str(value)) 185 elif isinstance(value, float): 186 return(str(value)) 187 elif value == None: 188 return("null") 189 elif isinstance(value, list): 190 try: 191 return(",".join(value)) 192 except TypeError: 193 raise TypeError("Cannot convert list on non-string objects to sql") 194 else: 195 raise TypeError(f"Cannot convert value of type {type(value)} to sql") 196 197def dict_to_sql(data: dict) -> str: 198 """Converts a dict into sql key value pairs. Ex: \"key1 = value1, key2 = value2...\"""" 199 200 set_list = [] 201 for field in data: 202 set_list.append(f"{field} = {value_to_sql_value(data[field])}") 203 return(", ".join(set_list)) 204 205 206class DatabaseError(Exception): 207 """Raised when the database fails to execute command""" 208 209class QueryError(Exception): 210 """Raised when trying to create an invalid or unsupperted query""" 211 212# TODO implement JOIN and LEFTJOIN (RIGHTJOIN?): https://www.w3schools.com/sql/sql_join.asp 213class Query: 214 """ 215 A class for writing sql queries. Queries can be run on the attached database or a seperate one with the `run` method. 216 217 Parameters 218 ---------- 219 db : Database, optional 220 The attached Database. This is the default database to run queries on. 221 verbose : bool, optional 222 Print what is going on in the `Query` 223 """ 224 225 def __init__(self, db=None, verbose=False) -> None: 226 227 self._db: Database = db 228 """The attached Database""" 229 230 self.sql = "" 231 """The current raw sql command""" 232 233 self.history = [] 234 """The history of commandmethods run on this object""" 235 236 self.fields = None 237 """The selected fields""" 238 239 self.table = None 240 """The table the sql query is interacting with""" 241 242 self.verbose = verbose 243 """Print what is going on in the `Query`""" 244 245 def valid_prefixes(self, prefixes: list) -> None: 246 """Check if a statement is valid given its prefix""" 247 248 prefix = None 249 if len(self.history) > 0: 250 prefix = self.history[-1] 251 if prefix in prefixes: 252 return(True) 253 raise QueryError(f"Query syntax incorrect or not supported. Prefix: \"{prefix}\" is not a part of the valid prefixes: {prefixes}") 254 255 def SELECT(self, selection="*"): 256 """ 257 Sql `SELECT` statement. Must be followed by `FROM` statement. 258 259 Parameters 260 ---------- 261 selection : str/list, optional 262 Either a python list or sql list of table names. Selects all columns if not set. 263 """ 264 265 self.valid_prefixes([None]) 266 self.history.append("SELECT") 267 268 if isinstance(selection, str): 269 if selection == "*": 270 self.fields = "*" 271 else: 272 self.fields = string_to_list(selection) 273 self.sql += f"SELECT {selection} " 274 elif isinstance(selection, list): 275 self.fields = selection 276 self.sql += f"SELECT {', '.join(selection)} " 277 else: 278 raise QueryError("SELECT statement selection must be either `str` or `list`") 279 return(self) 280 281 def FROM(self, table_name): 282 """ 283 Sql `FROM` statement. Has to be preceded by a SELECT statement. Can be followed by `WHERE` statement. 284 285 Parameters 286 ---------- 287 table_name : str 288 Name of the table you are selecting from. 289 """ 290 291 self.valid_prefixes(["SELECT"]) 292 self.table = table_name 293 294 if self._db: 295 table_fields = set(self._db.get_column_names(table_name)) # check if selected fields are in table 296 297 if self.fields != "*" and self._db and not set(self.fields).issubset(table_fields): 298 raise QueryError(f"Some selected field(s): {set(self.fields) - table_fields} are not fields/columns in the table: {table_name!r}. The table has the following fields: {table_fields}") 299 300 self.history.append("FROM") 301 self.sql += f"FROM {table_name} " 302 return(self) 303 304 def WHERE(self, col_name:str, value = ""): 305 """ 306 Sql `WHERE` statement. Can be followed by `LIKE` statement. 307 308 Parameters 309 ---------- 310 col_name : str 311 The name of the column. You can also just pass it a statement like: `"id" = 4` instead of providing a value. 312 value : optional 313 The value of the column. 314 """ 315 316 self.valid_prefixes(["FROM", "SET", "DELETE_FROM"]) 317 self.history.append("WHERE") 318 if value != "": 319 if value == None: 320 self.sql += f"WHERE {col_name} is null" 321 else: 322 self.sql += f"WHERE {col_name} = {value_to_sql_value(value)}" 323 else: 324 self.sql += f"WHERE {col_name} " 325 if col_name.find("=") == -1: # expects LIKE statement 326 self.col = col_name.replace(" ", "") 327 return(self) 328 329 def LIKE(self, pattern: str): 330 """ 331 Sql LIKE statement. Has to be preceded by a WHERE statement. 332 333 Parameters 334 ---------- 335 pattern : str 336 A typical sql LIKE pattern with % and _. 337 """ 338 339 self.valid_prefixes(["WHERE"]) 340 self.history.append("LIKE") 341 self.sql += f"LIKE {value_to_sql_value(pattern)} " 342 return(self) 343 344 def UPDATE(self, table_name: str): 345 """ 346 Sql UPDATE statement. Must be followed by `SET` statement. 347 348 Parameters 349 ---------- 350 table_name : str 351 Name of the table you are updating. 352 """ 353 354 self.valid_prefixes([None]) 355 self.history.append("UPDATE") 356 if self._db: 357 if not self._db.is_table(table_name): 358 raise QueryError(f"Database has no table called {table_name!r}") 359 self.fields = self._db.get_column_names(table_name) 360 self.table = table_name 361 self.sql += f"UPDATE {table_name} " 362 return(self) 363 364 def SET(self, data: dict): 365 """ 366 Sql SET statement. Must be preceded by an UPDATE statement. Must be followed by `WHERE` statement. 367 368 Parameters 369 ---------- 370 data : dict 371 A dictionaty with key and value pairs. 372 """ 373 374 self.valid_prefixes(["UPDATE"]) 375 self.history.append("SET") 376 377 data = dict(data) 378 379 if not set(data).issubset(self.fields): 380 raise DatabaseError(f"Data keys: {set(data)} are not a subset of table fields/columns. Table fields/columns: {set(self.fields)}") 381 382 self.sql += f"SET {dict_to_sql(data)} " 383 384 return(self) 385 386 def INSERT_INTO(self, table_name: str): 387 """ 388 Sql `INSERT INTO` statement. Must be followed by `VALUES` statement. 389 390 Parameters 391 ---------- 392 table_name : str 393 Name of the table you want to insert into. 394 """ 395 396 self.valid_prefixes([None]) 397 self.history.append("INSERT_INTO") 398 self.table = table_name 399 if self._db: 400 self.fields = self._db.get_column_names(table_name) 401 self.sql += f"INSERT INTO {table_name} " 402 return(self) 403 404 def VALUES(self, data: dict): 405 """ 406 Sql `VALUES` statement. Must be preceded by INSERT_INTO statement. 407 408 Parameters 409 ---------- 410 data : dict 411 Dictionary with key value pairs. 412 """ 413 414 self.valid_prefixes(["INSERT_INTO"]) 415 self.history.append("VALUES") 416 417 if not set(data).issubset(self.fields): 418 raise DatabaseError(f"Data keys: {set(data)} are not a subset of table fields/columns. Unknown keys: {set(data) - set(self.fields)}. Table fields/columns: {set(self.fields)}") 419 420 self.sql += f"({', '.join([str(v) for v in list(data)])}) VALUES ({', '.join([str(value_to_sql_value(v)) for v in data.values()])}) " 421 return(self) 422 423 def DELETE_FROM(self, table_name: str): 424 """ 425 Sql `DELETE FROM` statement. Must be followed by `WHERE` statement. 426 427 Parameters 428 ---------- 429 data : dict 430 Dictionary with key value pairs. 431 """ 432 433 self.valid_prefixes([None]) 434 self.history.append("DELETE_FROM") 435 if self._db and not table_name in self._db.get_table_names(): 436 raise QueryError(f"Can not perform DELETE FROM on a non-existing table: {table_name!r}") 437 self.table = table_name 438 self.sql = f"DELETE FROM {table_name} " 439 return(self) 440 441 442 def run(self, db=None, raw = False, verbose=False) -> list[DatabaseEntry]: 443 """ 444 Execute the query in the attached database or in a seperate one. Returns the results in a table (generator of DatabaseEntry) or `None` if no results. 445 446 Parameters 447 ---------- 448 db : Database, optional 449 The database to execute to query on. 450 raw : bool, optional 451 If True: returns the raw table (list of tuples) instead of the normal table. 452 verbose : bool, optional 453 Be verbose about it. 454 """ 455 456 if not db: 457 db = self._db 458 459 if not db: 460 raise DatabaseError("Query does not have a database to execute") 461 462 try: 463 db.cursor.execute(self.sql) 464 except sqlite3.OperationalError as e: 465 raise QueryError(f"\n\n{e}\n\nError while running following sql: {self.sql}") 466 467 if verbose or self.verbose or db.verbose: 468 print(f"Executed sql: {self.sql}") 469 470 results = db.cursor.fetchall() 471 472 if raw: 473 return(results) 474 475 if self.fields == "*": 476 self.fields = db.get_column_names(self.table) 477 478 return(raw_table_to_table(results, self.fields, self.table)) 479 480 def __repr__(self) -> str: 481 return(f"> {self.sql.strip()} <") 482 483 484 485 486class Database: 487 """ 488 Main database class for manipulating sqlite3 databases. 489 490 Parameters 491 ---------- 492 path : str 493 Path to the database file. 494 new : bool, optional 495 A new blank database will be created where the `self.path` is pointing. 496 verbose : bool, optional 497 Enables feedback in the form of prints. 498 """ 499 500 def __init__(self, path: str, new = False, verbose=False, silent=None): 501 502 if not new and not os.path.isfile(path): 503 raise(DatabaseError(f"No database file at \"{path}\". If you want to create one, pass \"new=True\"")) 504 505 self.path = path 506 """Path to the database file.""" 507 508 self.conn = sqlite3.connect(path) 509 """The sqlite3 connection.""" 510 511 self.cursor = self.conn.cursor() 512 """The sqlite3 cursor. Use `cursor.execute(cmd)` to execute raw sql.""" 513 514 self.connected: bool = True 515 """Is true if the `Database` instance is connected to a database.""" 516 517 self.verbose=verbose 518 """Enables feedback in the form of prints.""" 519 520 self.conn.execute("PRAGMA foreign_keys = ON") 521 522 # Deprecation notice 523 if isinstance(silent, bool): 524 print("[DEPRECATION] `silent` has been removed in favor of `verbose`. The `verbose` option is `False` by default.\n") 525 526 @classmethod 527 def in_memory(cls, verbose=False): 528 """ 529 Create a database in memory. Returns the `Database` instance. 530 531 Parameters 532 ---------- 533 verbose : bool, optional 534 Enables feedback in the form of prints. 535 """ 536 return Database(":memory:", new=True, verbose=verbose) 537 538 def create_table(self, name: str, cols: list[Column]): 539 """ 540 Creates a table in the Database. 541 542 Parameters 543 ---------- 544 name : str 545 Name of the new table. 546 cols : list[Column] 547 List of columns in the new table. 548 """ 549 550 sql = f"CREATE TABLE {name} (\n" 551 552 foreign_keys: list[ForeignKey] = [] 553 554 for col in cols: 555 sql += f"{col.name!r} {col.type}" 556 557 if col.primary_key: 558 sql += " PRIMARY KEY" 559 if col.not_null: 560 sql += " NOT NULL" 561 if col.default_value: 562 sql += f" DEFAULT {col.default_value!r}" 563 if col.foreign_key: 564 foreign_keys.append(col.foreign_key) 565 sql += ",\n" 566 567 for key in foreign_keys: 568 sql += f"FOREIGN KEY({key.from_col}) REFERENCES {key.table}({key.to_col}),\n" 569 570 if key.on_update: 571 sql = sql[:-2] + f"\nON UPDATE {key.on_update},\n" 572 573 if key.on_delete: 574 sql = sql[:-2] + f"\nON DELETE {key.on_delete},\n" 575 576 577 sql = sql[:-2] + "\n)" # remove last ",\n" 578 579 self.cursor.execute(sql) 580 581 def rename_table(self, current_name: str, new_name: str): 582 """ 583 Renames a table in the database. 584 585 Parameters 586 ---------- 587 current_name : str 588 Current name of a table. 589 new_name : str 590 New name of the table. 591 """ 592 self.cursor.execute(f"ALTER TABLE {current_name} RENAME TO {new_name}") 593 594 595 def delete_table(self, table_name: str) -> None: 596 """ 597 Deletes a table in the database. 598 599 Parameters 600 ---------- 601 table_name : Name of the table. 602 """ 603 self.cursor.execute(f"DROP TABLE {table_name}") 604 605 def add_column(self, table_name: str, col: Column): 606 """ 607 Add column to a table in the database. 608 609 Parameters 610 ---------- 611 table_name : str 612 Name of the table. 613 col : Column 614 The column to add to table. 615 """ 616 617 # Check that the table exists 618 if not self.is_table(table_name): 619 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 620 621 sql = f"ALTER TABLE {table_name} ADD COLUMN {col.name} {col.type}" 622 623 if col.primary_key: 624 sql += " PRIMARY KEY" 625 if col.not_null: 626 sql += " NOT NULL" 627 if col.default_value: 628 sql += f" DEFAULT {col.default_value}" 629 if col.foreign_key: 630 raise DatabaseError(f"Sqlite3 and therefore sqlite-integrated, does not support adding columns with foreign key constraings to existing tables. They have to be declared with the creation of the table.") 631 632 self.cursor.execute(sql) 633 634 def rename_column(self, table_name: str, current_column_name: str, new_column_name: str): 635 """ 636 Renames a column in the database. 637 638 Parameters 639 ---------- 640 table_name : str 641 Name of the table. 642 current_column_name : str 643 Current name of a column. 644 new_column_name : str 645 New name of the column. 646 """ 647 648 # Check that the table exists 649 if not self.is_table(table_name): 650 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 651 652 self.cursor.execute(f"ALTER TABLE {table_name} RENAME COLUMN {current_column_name} TO {new_column_name}") 653 654 def delete_column(self, table_name: str, col): 655 """ 656 Deletes a column in a table. 657 658 Parameters 659 ---------- 660 table_name : str 661 Name of the table the column is in. 662 col : str/Column 663 Column, or column name, of the column that should be deleted. 664 """ 665 666 # Check that the table exists 667 if not self.is_table(table_name): 668 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 669 670 if col is Column: 671 col = col.name 672 673 self.cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN {col}") 674 675 676 677 def get_table_names(self) -> list: 678 """Returns the names of all tables in the database.""" 679 680 res = self.conn.execute("SELECT name FROM sqlite_master WHERE type='table';") 681 names = [] 682 for name in res: 683 names.append(name[0]) 684 return(names) 685 686 def is_table(self, table_name: str) -> bool: 687 """ 688 Check if database has a table with a certain name. 689 690 Parameters 691 ---------- 692 table_name : str 693 Name to check. 694 695 """ 696 697 if table_name in self.get_table_names(): 698 return True 699 return False 700 701 def get_table_raw(self, name: str, get_only = None) -> list: 702 """ 703 Returns all entries in a table as a list of tuples. 704 705 Parameters 706 ---------- 707 name : str 708 Name of the table. 709 get_only : list, optional 710 Can be set to a list of column/field names, to only retrieve those columns/fields. 711 """ 712 713 selected = "*" 714 715 if get_only: 716 if isinstance(get_only, list): 717 fields = self.get_column_names(name) 718 for field in get_only: 719 if not field in fields: 720 raise DatabaseError(f"Table \"{name}\" contains no field/column with the name: \"{field}\". Available fields are: {fields}") 721 selected = ','.join(get_only) 722 else: 723 raise ValueError(f"get_only can either be `None` or `list`. Got: {get_only}") 724 725 self.cursor.execute(f"SELECT {selected} FROM {name}") 726 return(self.cursor.fetchall()) 727 728 def get_table(self, name: str, get_only=None) -> list: 729 """ 730 Returns all entries in a table as a table (generator of DatabaseEntry). This function loops over all entries in the table, so it is not the best in very big databases. 731 732 Parameters 733 ---------- 734 name : str 735 Name of the table. 736 get_only : list/None, optional 737 Can be set to a list of column/field names, to only retrieve those columns/fields. 738 """ 739 740 raw_table = self.get_table_raw(name, get_only) 741 742 return(raw_table_to_table(raw_table, self.get_column_names(name), name)) 743 744 745 def get_table_cols(self, name: str) -> list[Column]: 746 """ 747 Returns a list of Column objects, that contain information about the table columns. 748 749 Parameters 750 ---------- 751 name : str 752 Name of the table. 753 """ 754 755 self.cursor.execute(f"PRAGMA table_info({name});") 756 cols_raw_info = self.cursor.fetchall() 757 758 cols = [] 759 for col_raw_info in cols_raw_info: 760 is_primary = False 761 if col_raw_info[5] == 1: 762 is_primary = True 763 not_null = False 764 if col_raw_info[3] == 1: 765 not_null = True 766 cols.append(Column(col_raw_info[1], col_raw_info[2], not_null, col_raw_info[4], is_primary, col_id=col_raw_info[0])) 767 768 769 # Add foreign keys to cols 770 self.cursor.execute(f"PRAGMA foreign_key_list({name});") 771 foreign_key_list = self.cursor.fetchall() 772 773 if len(foreign_key_list) > 0: 774 for raw_foreign_key in foreign_key_list: 775 foreign_key = ForeignKey( 776 raw_foreign_key[2], 777 raw_foreign_key[4], 778 id=raw_foreign_key[0], 779 seq=raw_foreign_key[1], 780 from_col=raw_foreign_key[3], 781 on_update=raw_foreign_key[5], 782 on_delete=raw_foreign_key[6], 783 match=raw_foreign_key[7] 784 ) 785 786 for n, col in enumerate(cols): 787 if col.name == foreign_key.from_col: 788 cols[n].foreign_key = foreign_key 789 break 790 return(cols) 791 792 def get_table_id_field(self, table: str, do_error=False) -> str: 793 """ 794 Takes a table and returns the name of the field/column marked as a `PRIMARY KEY`. (This function assumes that there is only ONE field marked as a `PRIMARY KEY`). 795 796 Parameters 797 ---------- 798 table : str 799 Name of the table. 800 do_error : bool, optional 801 If True: Raises error if the table does not contain a field marked as `PRIMARY KEY`. 802 """ 803 804 cols = self.get_table_cols(table) 805 806 for col in cols: 807 if col.primary_key == True: # col_info[5] is 1 if field is a primary key. Otherwise it is 0. 808 return col.name # col_info[1] is the name of the column 809 if do_error: 810 raise DatabaseError(f"The table `{table}` has no id_field (column defined as a `PRIMARY KEY`)") 811 return(None) 812 813 def table_overview(self, name: str, max_len:int = 40, get_only = None) -> None: 814 """ 815 Prints a pretty table (with a name). 816 817 Parameters 818 ---------- 819 name : str 820 Name of the table. 821 max_len : int, optional 822 The max number of rows shown. 823 get_only : list, optional 824 If given a list of column/field names: only shows those. 825 826 """ 827 828 text = "" # the output text 829 830 raw_table = self.get_table_raw(name, get_only=get_only) 831 832 if get_only: 833 fields = get_only 834 else: 835 fields = self.get_column_names(name) 836 837 cols = len(fields) 838 839 longest_words = [0] * cols 840 841 words_table = raw_table + [fields] 842 843 844 for col in range(cols): 845 for entry in words_table: 846 if len(str(entry[col])) > longest_words[col]: 847 longest_words[col] = len(str(entry[col])) 848 849 seperator = " ║ " 850 851 def formatRow(row, longest_words): 852 formatted_list = [] 853 for i, string in enumerate(row): 854 string = str(string) 855 formatted_list.append(string + " " * (longest_words[i] - len(string))) 856 return(seperator.join(formatted_list)) 857 858 text += formatRow(fields, longest_words) + "\n" 859 underline = "═" * (sum(longest_words) + len(seperator)) 860 861 # This block is for placing the intersections 862 offset = 0 863 for n in longest_words[:-1]: # we dont create the an intersection after the last column 864 offset += n 865 underline = underline[:offset +1] + "╬" + underline[offset:] 866 offset += len(seperator) 867 868 text += underline + "\n" 869 870 if len(raw_table) >= max_len: 871 for row in raw_table[:max_len-5]: 872 text += formatRow(row, longest_words) + "\n" 873 text += " .\n .\n .\n" 874 for row in raw_table[-5:]: 875 text += formatRow(row, longest_words) + "\n" 876 else: 877 for row in raw_table: 878 text += formatRow(row, longest_words) + "\n" 879 880 print(text) 881 882 def overview(self, more=False) -> None: 883 """ 884 Prints an overview of all the tables in the database with their fields. 885 886 Parameters 887 ---------- 888 more : optional 889 If true: Prints more information on the columns in each table. 890 """ 891 892 table_names = self.get_table_names() 893 894 # if there are no tables in database 895 if len(table_names) == 0: 896 print(f"There are no tables in sqlite database at \"{self.path}\".") 897 return(None) 898 899 text = "Tables\n" 900 for table_name in table_names: 901 text += "\t" + table_name + "\n" 902 for col in self.get_table_cols(table_name): 903 text += f"\t\t{col.name}" 904 if more: 905 text += f"\t\t[{col}]" 906 text += "\n" 907 print(text) 908 909 910 def get_column_names(self, table_name: str) -> list[str]: 911 """ 912 Returns the field/column names for a given table. 913 914 Parameters 915 ---------- 916 table_name : str 917 Name of the table. 918 """ 919 920 if not self.is_table(table_name): 921 raise DatabaseError(f"Can not get column names of non-existing table {table_name!r}.") 922 923 names = [] 924 925 for col in self.get_table_cols(table_name): 926 names.append(col.name) 927 return(names) 928 929 def is_column(self, table_name: str, col_name: str) -> bool: 930 """ 931 Returns True if the given column name exists in the given table. Else returns False. 932 933 Parameters 934 ---------- 935 table_name : str 936 Name of a table. 937 col_name : str 938 Name of a column that may be in the table. 939 """ 940 941 if col_name in self.get_column_names(table_name): 942 return(True) 943 return(False) 944 945 def fill_null(self, entry: DatabaseEntry) -> DatabaseEntry: 946 """ 947 Fills out any unpopulated fields in a DatabaseEntry (fields that exist in the database table but not in the entry) and returns it. 948 949 Parameters 950 ---------- 951 entry : DatabaseEntry 952 The DatabaseEntry. 953 """ 954 955 t_fields = self.get_column_names(entry.table) 956 e_fields = list(entry) 957 for f in e_fields: 958 t_fields.remove(f) 959 for null_field in t_fields: 960 entry[null_field] = None 961 return(entry) 962 963 964 def get_entry_by_id(self, table, ID) -> DatabaseEntry: 965 """ 966 Get table entry by id. 967 968 Parameters 969 ---------- 970 table : str 971 Name of the table. 972 ID : 973 The entry id. 974 """ 975 976 id_field = self.get_table_id_field(table, do_error=True) 977 978 if not self.is_table(table): 979 raise DatabaseError(f"Database contains no table with the name: \"{table}\". These are the available tables: {self.get_table_names()}") 980 981 sql = f"SELECT * FROM {table} WHERE {id_field} = {ID}" 982 983 self.cursor.execute(sql) 984 985 answer = self.cursor.fetchall() 986 987 # some checks 988 if len(answer) != 1: 989 if len(answer) > 1: 990 raise DatabaseError(f"There are more than one entry in table \"{table}\" with an id field \"{id_field}\" with the value \"{id}\": {answer}") 991 elif len(answer) == 0: 992 raise DatabaseError(f"There is no entry in table \"{table}\" with an id_field \"{id_field}\" with a value of {ID}") 993 else: 994 raise DatabaseError("Something went very wrong, please contact the package author") # this will never be run... i think 995 996 return(DatabaseEntry.from_raw_entry(answer[0], self.get_column_names(table), table)) 997 998 def add_entry(self, entry, table = None, fill_null=False, verbose=False) -> None: 999 """ 1000 Add an entry to the database by passing a DatabaseEntry, or with a dictionary and specifying a table name. 1001 1002 Returns the id of the added DatabaseEntry in the table, or `None` if table does not contain a primary key. 1003 1004 The entry must have values for all fields in the table. You can pass `fill_null=True` to fill any remaining fields with `None`/`null`. 1005 1006 Parameters 1007 ---------- 1008 entry : DatabaseEntry/dict 1009 The entry. 1010 table : str, optional 1011 Name of the table the entry belongs to. **Needed if adding an entry with a dictionary**. 1012 fill_null : bool, optional 1013 Fill in unpopulated fields with null values. 1014 verbose : bool, optional 1015 Enable prints. 1016 """ 1017 1018 if type(entry) == dict: 1019 if not table: 1020 raise DatabaseError(f"Please provide the table that the data should be inserted in.") 1021 entry = DatabaseEntry(entry, table) 1022 1023 if not self.is_table(entry.table): 1024 raise DatabaseError(f"Database has no table with the name \"{self.table}\". Possible tablenames are: {self.get_table_names()}") 1025 1026 table_fields = self.get_column_names(entry.table) 1027 1028 id_field = self.get_table_id_field(entry.table) 1029 1030 if id_field: 1031 entry[id_field] = None 1032 1033 if fill_null: 1034 entry = self.fill_null(entry) 1035 1036 if set(entry) != set(table_fields): 1037 raise DatabaseError(f"entry fields are not the same as the table fields: {set(entry)} != {set(table_fields)}") 1038 1039 self.INSERT_INTO(entry.table).VALUES(entry).run() 1040 1041 if verbose or self.verbose: 1042 print(f"added entry to table \"{entry.table}\": {entry}") 1043 1044 if not self.get_table_id_field(table): 1045 return None 1046 1047 self.cursor.execute("SELECT last_insert_rowid()") 1048 return (self.cursor.fetchall()[0][0]) 1049 1050 1051 def update_entry(self, entry: dict, table=None, part=False, fill_null=False, verbose=False) -> None: 1052 """ 1053 Update entry in database with a DatabaseEntry, or with a dictionary + the name of the table you want to update. 1054 1055 Parameters 1056 ---------- 1057 entry : DatabaseEntry/dict 1058 DatabaseEntry or dictionary, if dictionary you also need to provide table and id_field. 1059 table : str, optional 1060 The table name. **Needed if updating an entry with a dictionary**. 1061 part : bool, optional 1062 If True: Only updates the provided fields. 1063 fill_null : bool, optional 1064 Fill in unpopulated fields with null values. 1065 verbose : bool, optional 1066 Enable prints. 1067 """ 1068 1069 if not isinstance(entry, DatabaseEntry): # the input is a dict 1070 if not table: 1071 raise DatabaseError(f"Please provide a table when updating an entry with a python dictionary") 1072 entry = DatabaseEntry(entry, table) 1073 1074 id_field = self.get_table_id_field(entry.table) 1075 1076 if not self.is_table(entry.table): 1077 raise DatabaseError(f"Database has no table with the name \"{entry.table}\". Possible tablenames are: {self.get_table_names()}") 1078 1079 if fill_null: 1080 entry = self.fill_null(entry) 1081 1082 # check that entry fields and table fields match 1083 table_fields = self.get_column_names(entry.table) 1084 if set(table_fields) != set(entry): 1085 if not (part and set(entry).issubset(set(table_fields))): 1086 raise DatabaseError(f"Table fields do not match entry fields: {table_fields} != {list(entry)}. Pass `part = True` or `fill_null = True` if entry are a subset of the table fields") 1087 1088 self.UPDATE(entry.table).SET(entry).WHERE(id_field, entry[id_field]).run() 1089 1090 if verbose or self.verbose: 1091 print(f"updated entry in table \"{entry.table}\": {entry}") 1092 1093 def delete_entry(self, entry: DatabaseEntry): 1094 """ 1095 Delete an entry from the database. 1096 1097 Parameters 1098 ---------- 1099 entry : DatabaseEntry 1100 The entry that is to be deleted. 1101 """ 1102 1103 id_field = self.get_table_id_field(entry.table) 1104 self.DELETE_FROM(entry.table).WHERE(id_field, entry[id_field]).run() 1105 1106 1107 def delete_entry_by_id(self, table: str, id: int): 1108 """ 1109 Deletes an entry with a certain id. (Note: the table must have a primary key column, as that is what is meant by id. It is assumed that there is only one primary key column in the table.} 1110 1111 Parameters 1112 ---------- 1113 table : str 1114 The table to delete the entry from. 1115 id : int 1116 1117 """ 1118 1119 id_field = self.get_table_id_field(table) 1120 self.DELETE_FROM(table).WHERE(id_field, id).run() 1121 1122 def save(self) -> None: 1123 """Writes any changes to the database file""" 1124 1125 self.conn.commit() 1126 1127 def close(self) -> None: 1128 """Saves and closes the database. If you want to explicitly close without saving use: `self.conn.close()`""" 1129 1130 self.conn.commit() 1131 self.conn.close() 1132 self.connected = False 1133 1134 def reconnect(self) -> None: 1135 """Reopen database after closing it""" 1136 1137 self.conn = sqlite3.connect(self.path) 1138 self.cursor = self.conn.cursor() 1139 self.connected = True 1140 1141 def delete_table(self, table_name) -> None: 1142 """ 1143 Takes a table name and deletes the table from the database. 1144 1145 Parameters 1146 ---------- 1147 table_name : str 1148 Name of the table. 1149 """ 1150 1151 self.cursor.execute(f"DROP TABLE {table_name};") 1152 1153 def table_to_dataframe(self, table) -> pd.DataFrame: 1154 """ 1155 Converts a table to a pandas.Dataframe. 1156 1157 Parameters 1158 ---------- 1159 table : str 1160 Name of the table. 1161 """ 1162 1163 cols = {} 1164 fields = self.get_column_names(table) 1165 1166 for f in fields: 1167 cols[f] = [] 1168 1169 for raw_entry in self.get_table_raw(table): 1170 for n, field in enumerate(fields): 1171 cols[field].append(raw_entry[n]) 1172 1173 return(pd.DataFrame(cols)) 1174 1175 1176 def export_to_csv(self, out_dir: str, tables: list = None, sep: str = "\t") -> None: 1177 """ 1178 Export all or some tables in the database to csv files 1179 1180 Parameters 1181 ---------- 1182 out_dir : str 1183 Path to the output directory. 1184 tables : list[str]/None, optional 1185 Can be set to only export certain tables. 1186 sep : str, optional 1187 Seperator to use when writing csv-file. 1188 """ 1189 1190 if not os.path.isdir(out_dir): 1191 raise NotADirectoryError(f"{out_dir!r} is not a directory") 1192 1193 if not tables: 1194 tables = self.get_table_names() 1195 1196 for table_name in tables: 1197 df = self.table_to_dataframe(table_name) 1198 df.to_csv(f"{out_dir}/{table_name}.csv", index=False, sep=sep) 1199 1200 def run_raw_sql(self, sql: str, verbose=False): 1201 """ 1202 Run SQL-string on the database. This returns a raw table as list of tuples. 1203 1204 Parameters 1205 ---------- 1206 sql : str 1207 SQL-string to be execured as an SQL command. 1208 verbose : bool, optional 1209 Prints the SQL-query if true 1210 """ 1211 1212 try: 1213 self.cursor.execute(sql) 1214 except sqlite3.OperationalError as e: 1215 raise QueryError(f"\n\n{e}\n\nError while running following sql: {self.sql}") 1216 1217 if verbose or self.verbose: 1218 print(f"Executed sql: {self.sql}") 1219 1220 return(self.cursor.fetchall()) 1221 1222 def SELECT(self, pattern="*") -> Query: 1223 """ 1224 Start sql SELECT query from the database. Returns a Query to build from. 1225 1226 Parameters 1227 ---------- 1228 pattern : str, optional 1229 Either a python list or sql list of table names. 1230 """ 1231 1232 return(Query(db=self).SELECT(pattern)) 1233 1234 def UPDATE(self, table_name) -> Query: 1235 """ 1236 Start sql UPDATE query from the database. Returns a Query to build from. 1237 1238 Parameters 1239 ---------- 1240 table_name : str 1241 Name of the table. 1242 """ 1243 return(Query(db=self).UPDATE(table_name)) 1244 1245 def INSERT_INTO(self, table_name) -> Query: 1246 """ 1247 Start sql INSERT INTO query from the database. Returns a Query to build from. 1248 1249 Parameters 1250 ---------- 1251 table_name : str 1252 Name of the table to insert into. 1253 """ 1254 1255 return(Query(db=self).INSERT_INTO(table_name)) 1256 1257 def DELETE_FROM(self, table_name: str) -> Query: 1258 """ 1259 Start sql DELETE FROM query from the database. Returns a Query to build from. 1260 1261 Parameters 1262 ---------- 1263 table_name : str 1264 Name of the table to delete from. 1265 """ 1266 return(Query(db=self).DELETE_FROM(table_name)) 1267 1268 1269 def __eq__(self, other: object) -> bool: 1270 tables = self.get_table_names() 1271 if tables != other.get_table_names(): 1272 return(False) 1273 1274 for table in tables: 1275 if self.get_table_raw(table) != other.get_table_raw(table): 1276 return(False) 1277 elif self.get_table_cols(table) != other.get_table_cols(table): 1278 return(False) 1279 return(True)
8@dataclass 9class ForeignKey: 10 """Class representing an sql foreign key""" 11 12 table: str 13 """The table the foreign key points to""" 14 15 to_col: str 16 """Column the foreign key points to""" 17 18 from_col: str = None 19 """Column in current table, containing the key value""" 20 21 id: int = None 22 """The foreign key id""" 23 24 seq: int = None 25 """The foreign key sequence attribute""" 26 27 on_update: str = None 28 """The action the column will do if the data the key is pointing to changes. (Provide sql action).""" 29 30 on_delete: str = None 31 """The action the column will do if the data the key is pointing to changes. (Provide sql action).""" 32 33 match: str = None 34 35 def to_sql(self): 36 rep = f"FOREIGN KEY ({self.from_col}) REFERENCES {self.table} ({self.to_col})" 37 if self.on_update: 38 rep += f" ON UPDATE {self.on_update}" 39 if self.on_delete: 40 rep += f" ON DELETE {self.on_delete}" 41 return(rep)
Class representing an sql foreign key
The action the column will do if the data the key is pointing to changes. (Provide sql action).
43@dataclass 44class Column: 45 """Class representing en sql column.""" 46 47 def __init__(self, name: str, type: str, not_null: bool = None, default_value: any = None, primary_key: bool = False, col_id: int = None, foreign_key: ForeignKey = None) -> None: 48 49 if primary_key and type.upper() != "INTEGER": 50 raise DatabaseError(f"Primary key columns must have sqlite type: `INTEGER` not \'{type}\'") 51 52 self.name = name 53 """Name of the column.""" 54 55 self.type = type 56 """Type of the data in the column.""" 57 58 self.not_null = not_null 59 """Sql NOT NULL constraint.""" 60 61 self.default_value = default_value 62 """Sql DEFAULT. Default value for the column.""" 63 64 self.primary_key = primary_key 65 """Sql PRIMARY KEY. Automatic column that ensures that every entry has a unique.""" 66 67 self.col_id = col_id 68 """Id if the column in the table.""" 69 70 if foreign_key: 71 foreign_key.from_col = name 72 73 self.foreign_key = foreign_key 74 """ForeignKey object, that representing an sql foreign key.""" 75 76 77 def __repr__(self) -> str: 78 attrs = [] 79 if self.col_id: 80 attrs.append(str(self.col_id)) 81 attrs.append(self.name) 82 attrs.append(self.type) 83 if self.not_null: 84 attrs.append("NOT NULL") 85 if self.default_value: 86 attrs.append(f"DEFAULT: {self.default_value}") 87 if self.primary_key: 88 attrs.append("PRIMARY KEY") 89 if self.foreign_key: 90 attrs.append(self.foreign_key.to_sql()) 91 return(f"Column({', '.join(attrs)})")
Class representing en sql column.
47 def __init__(self, name: str, type: str, not_null: bool = None, default_value: any = None, primary_key: bool = False, col_id: int = None, foreign_key: ForeignKey = None) -> None: 48 49 if primary_key and type.upper() != "INTEGER": 50 raise DatabaseError(f"Primary key columns must have sqlite type: `INTEGER` not \'{type}\'") 51 52 self.name = name 53 """Name of the column.""" 54 55 self.type = type 56 """Type of the data in the column.""" 57 58 self.not_null = not_null 59 """Sql NOT NULL constraint.""" 60 61 self.default_value = default_value 62 """Sql DEFAULT. Default value for the column.""" 63 64 self.primary_key = primary_key 65 """Sql PRIMARY KEY. Automatic column that ensures that every entry has a unique.""" 66 67 self.col_id = col_id 68 """Id if the column in the table.""" 69 70 if foreign_key: 71 foreign_key.from_col = name 72 73 self.foreign_key = foreign_key 74 """ForeignKey object, that representing an sql foreign key."""
94class DatabaseEntry(dict): 95 """ 96 A python dictionary that keeps track of the table it belongs to. This class is not meant to be created manually. 97 98 Parameters 99 ---------- 100 entry_dict : dict 101 A dictionary containing all the information. This information can be accesed just like any other python dict with `my_entry[my_key]`. 102 table : str 103 The name of the table the entry is a part of 104 """ 105 106 def __init__(self, entry_dict: dict, table: str): 107 self.table = table 108 self.update(entry_dict) 109 110 111 @classmethod 112 def from_raw_entry(cls, raw_entry: tuple, table_fields: list, table_name: str): 113 """ 114 Alternative constructor for converting a raw entry to a DatabaseEntry. 115 116 Parameters 117 ---------- 118 raw_entry : tuple 119 A tuple with the data for the entry. Ex: `(2, "Tom", "Builder", 33)` 120 table_fields : list 121 A list of column names for the data. Ex: `["id", "FirstName", "LastName", "Age"]` 122 table_name : str 123 The name of the table (in the database) that the data belongs to. Ex: "people" 124 """ 125 126 entry_dict = {} 127 128 if isinstance(table_fields, str): 129 table_fields = string_to_list(table_fields) 130 elif not isinstance(table_fields, list): 131 raise ValueError(f"table_fields must be either `list` or `str`. Got: {table_fields}") 132 133 if len(raw_entry) != len(table_fields): 134 raise DatabaseError(f"There must be as many names for table fields as there are fields in the entry: len({raw_entry}) != len({table_fields}) => {len(raw_entry)} != {len(table_fields)}") 135 136 for n, field in enumerate(table_fields): 137 entry_dict[field] = raw_entry[n] 138 entry = DatabaseEntry(entry_dict, table_name) 139 return(entry) 140 141 142 def __repr__(self) -> str: 143 """Represent a Database entry""" 144 145 return f"DatabaseEntry(table: {self.table}, data: {super().__repr__()})"
A python dictionary that keeps track of the table it belongs to. This class is not meant to be created manually.
Parameters
entry_dict : dict
A dictionary containing all the information. This information can be accesed just like any other python dict with my_entry[my_key]
.
table : str
The name of the table the entry is a part of
111 @classmethod 112 def from_raw_entry(cls, raw_entry: tuple, table_fields: list, table_name: str): 113 """ 114 Alternative constructor for converting a raw entry to a DatabaseEntry. 115 116 Parameters 117 ---------- 118 raw_entry : tuple 119 A tuple with the data for the entry. Ex: `(2, "Tom", "Builder", 33)` 120 table_fields : list 121 A list of column names for the data. Ex: `["id", "FirstName", "LastName", "Age"]` 122 table_name : str 123 The name of the table (in the database) that the data belongs to. Ex: "people" 124 """ 125 126 entry_dict = {} 127 128 if isinstance(table_fields, str): 129 table_fields = string_to_list(table_fields) 130 elif not isinstance(table_fields, list): 131 raise ValueError(f"table_fields must be either `list` or `str`. Got: {table_fields}") 132 133 if len(raw_entry) != len(table_fields): 134 raise DatabaseError(f"There must be as many names for table fields as there are fields in the entry: len({raw_entry}) != len({table_fields}) => {len(raw_entry)} != {len(table_fields)}") 135 136 for n, field in enumerate(table_fields): 137 entry_dict[field] = raw_entry[n] 138 entry = DatabaseEntry(entry_dict, table_name) 139 return(entry)
Alternative constructor for converting a raw entry to a DatabaseEntry.
Parameters
raw_entry : tuple
A tuple with the data for the entry. Ex: (2, "Tom", "Builder", 33)
table_fields : list
A list of column names for the data. Ex: ["id", "FirstName", "LastName", "Age"]
table_name : str
The name of the table (in the database) that the data belongs to. Ex: "people"
Inherited Members
- builtins.dict
- get
- setdefault
- pop
- popitem
- keys
- items
- values
- update
- fromkeys
- clear
- copy
148def raw_table_to_table(raw_table: list, fields: list, table_name: str) -> list[DatabaseEntry]: 149 """ 150 Convert a raw table (list of tuples) to a table (generator of DatabaseEntry). 151 152 Parameters 153 ---------- 154 raw_table : list 155 A list of tuples with the data for the entries. 156 fields : list 157 A list of column names for the data. Ex: `["id", "FirstName", "LastName", "Age"]` 158 table_name: str 159 The name of the table (in the database) that the data belongs to. Ex: "people". 160 """ 161 162 if len(raw_table) == 0: 163 return 164 if len(raw_table[0]) != len(fields): 165 raise DatabaseError(f"There must be one raw column per field. {raw_table[0] = }, {fields = }") 166 167 for raw_entry in raw_table: 168 entry = {} 169 for n, field in enumerate(fields): 170 entry[field] = raw_entry[n] 171 yield DatabaseEntry(entry, table_name)
Convert a raw table (list of tuples) to a table (generator of DatabaseEntry).
Parameters
raw_table : list
A list of tuples with the data for the entries.
fields : list
A list of column names for the data. Ex: ["id", "FirstName", "LastName", "Age"]
table_name: str
The name of the table (in the database) that the data belongs to. Ex: "people".
174def string_to_list(string: str) -> list: 175 """Takes a string with comma seperated values, returns a list of the values. (spaces are ignored)""" 176 177 return(string.replace(" ", "").split(","))
Takes a string with comma seperated values, returns a list of the values. (spaces are ignored)
179def value_to_sql_value(value) -> str: 180 """Converts python values to sql values. Basically just puts quotes around strings and not ints or floats. Also converts None to null""" 181 182 if isinstance(value, str): 183 return("'" + value.replace("'", "''") + "'") 184 elif isinstance(value, int): 185 return(str(value)) 186 elif isinstance(value, float): 187 return(str(value)) 188 elif value == None: 189 return("null") 190 elif isinstance(value, list): 191 try: 192 return(",".join(value)) 193 except TypeError: 194 raise TypeError("Cannot convert list on non-string objects to sql") 195 else: 196 raise TypeError(f"Cannot convert value of type {type(value)} to sql")
Converts python values to sql values. Basically just puts quotes around strings and not ints or floats. Also converts None to null
198def dict_to_sql(data: dict) -> str: 199 """Converts a dict into sql key value pairs. Ex: \"key1 = value1, key2 = value2...\"""" 200 201 set_list = [] 202 for field in data: 203 set_list.append(f"{field} = {value_to_sql_value(data[field])}") 204 return(", ".join(set_list))
Converts a dict into sql key value pairs. Ex: "key1 = value1, key2 = value2..."
Raised when the database fails to execute command
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- add_note
210class QueryError(Exception): 211 """Raised when trying to create an invalid or unsupperted query"""
Raised when trying to create an invalid or unsupperted query
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- add_note
214class Query: 215 """ 216 A class for writing sql queries. Queries can be run on the attached database or a seperate one with the `run` method. 217 218 Parameters 219 ---------- 220 db : Database, optional 221 The attached Database. This is the default database to run queries on. 222 verbose : bool, optional 223 Print what is going on in the `Query` 224 """ 225 226 def __init__(self, db=None, verbose=False) -> None: 227 228 self._db: Database = db 229 """The attached Database""" 230 231 self.sql = "" 232 """The current raw sql command""" 233 234 self.history = [] 235 """The history of commandmethods run on this object""" 236 237 self.fields = None 238 """The selected fields""" 239 240 self.table = None 241 """The table the sql query is interacting with""" 242 243 self.verbose = verbose 244 """Print what is going on in the `Query`""" 245 246 def valid_prefixes(self, prefixes: list) -> None: 247 """Check if a statement is valid given its prefix""" 248 249 prefix = None 250 if len(self.history) > 0: 251 prefix = self.history[-1] 252 if prefix in prefixes: 253 return(True) 254 raise QueryError(f"Query syntax incorrect or not supported. Prefix: \"{prefix}\" is not a part of the valid prefixes: {prefixes}") 255 256 def SELECT(self, selection="*"): 257 """ 258 Sql `SELECT` statement. Must be followed by `FROM` statement. 259 260 Parameters 261 ---------- 262 selection : str/list, optional 263 Either a python list or sql list of table names. Selects all columns if not set. 264 """ 265 266 self.valid_prefixes([None]) 267 self.history.append("SELECT") 268 269 if isinstance(selection, str): 270 if selection == "*": 271 self.fields = "*" 272 else: 273 self.fields = string_to_list(selection) 274 self.sql += f"SELECT {selection} " 275 elif isinstance(selection, list): 276 self.fields = selection 277 self.sql += f"SELECT {', '.join(selection)} " 278 else: 279 raise QueryError("SELECT statement selection must be either `str` or `list`") 280 return(self) 281 282 def FROM(self, table_name): 283 """ 284 Sql `FROM` statement. Has to be preceded by a SELECT statement. Can be followed by `WHERE` statement. 285 286 Parameters 287 ---------- 288 table_name : str 289 Name of the table you are selecting from. 290 """ 291 292 self.valid_prefixes(["SELECT"]) 293 self.table = table_name 294 295 if self._db: 296 table_fields = set(self._db.get_column_names(table_name)) # check if selected fields are in table 297 298 if self.fields != "*" and self._db and not set(self.fields).issubset(table_fields): 299 raise QueryError(f"Some selected field(s): {set(self.fields) - table_fields} are not fields/columns in the table: {table_name!r}. The table has the following fields: {table_fields}") 300 301 self.history.append("FROM") 302 self.sql += f"FROM {table_name} " 303 return(self) 304 305 def WHERE(self, col_name:str, value = ""): 306 """ 307 Sql `WHERE` statement. Can be followed by `LIKE` statement. 308 309 Parameters 310 ---------- 311 col_name : str 312 The name of the column. You can also just pass it a statement like: `"id" = 4` instead of providing a value. 313 value : optional 314 The value of the column. 315 """ 316 317 self.valid_prefixes(["FROM", "SET", "DELETE_FROM"]) 318 self.history.append("WHERE") 319 if value != "": 320 if value == None: 321 self.sql += f"WHERE {col_name} is null" 322 else: 323 self.sql += f"WHERE {col_name} = {value_to_sql_value(value)}" 324 else: 325 self.sql += f"WHERE {col_name} " 326 if col_name.find("=") == -1: # expects LIKE statement 327 self.col = col_name.replace(" ", "") 328 return(self) 329 330 def LIKE(self, pattern: str): 331 """ 332 Sql LIKE statement. Has to be preceded by a WHERE statement. 333 334 Parameters 335 ---------- 336 pattern : str 337 A typical sql LIKE pattern with % and _. 338 """ 339 340 self.valid_prefixes(["WHERE"]) 341 self.history.append("LIKE") 342 self.sql += f"LIKE {value_to_sql_value(pattern)} " 343 return(self) 344 345 def UPDATE(self, table_name: str): 346 """ 347 Sql UPDATE statement. Must be followed by `SET` statement. 348 349 Parameters 350 ---------- 351 table_name : str 352 Name of the table you are updating. 353 """ 354 355 self.valid_prefixes([None]) 356 self.history.append("UPDATE") 357 if self._db: 358 if not self._db.is_table(table_name): 359 raise QueryError(f"Database has no table called {table_name!r}") 360 self.fields = self._db.get_column_names(table_name) 361 self.table = table_name 362 self.sql += f"UPDATE {table_name} " 363 return(self) 364 365 def SET(self, data: dict): 366 """ 367 Sql SET statement. Must be preceded by an UPDATE statement. Must be followed by `WHERE` statement. 368 369 Parameters 370 ---------- 371 data : dict 372 A dictionaty with key and value pairs. 373 """ 374 375 self.valid_prefixes(["UPDATE"]) 376 self.history.append("SET") 377 378 data = dict(data) 379 380 if not set(data).issubset(self.fields): 381 raise DatabaseError(f"Data keys: {set(data)} are not a subset of table fields/columns. Table fields/columns: {set(self.fields)}") 382 383 self.sql += f"SET {dict_to_sql(data)} " 384 385 return(self) 386 387 def INSERT_INTO(self, table_name: str): 388 """ 389 Sql `INSERT INTO` statement. Must be followed by `VALUES` statement. 390 391 Parameters 392 ---------- 393 table_name : str 394 Name of the table you want to insert into. 395 """ 396 397 self.valid_prefixes([None]) 398 self.history.append("INSERT_INTO") 399 self.table = table_name 400 if self._db: 401 self.fields = self._db.get_column_names(table_name) 402 self.sql += f"INSERT INTO {table_name} " 403 return(self) 404 405 def VALUES(self, data: dict): 406 """ 407 Sql `VALUES` statement. Must be preceded by INSERT_INTO statement. 408 409 Parameters 410 ---------- 411 data : dict 412 Dictionary with key value pairs. 413 """ 414 415 self.valid_prefixes(["INSERT_INTO"]) 416 self.history.append("VALUES") 417 418 if not set(data).issubset(self.fields): 419 raise DatabaseError(f"Data keys: {set(data)} are not a subset of table fields/columns. Unknown keys: {set(data) - set(self.fields)}. Table fields/columns: {set(self.fields)}") 420 421 self.sql += f"({', '.join([str(v) for v in list(data)])}) VALUES ({', '.join([str(value_to_sql_value(v)) for v in data.values()])}) " 422 return(self) 423 424 def DELETE_FROM(self, table_name: str): 425 """ 426 Sql `DELETE FROM` statement. Must be followed by `WHERE` statement. 427 428 Parameters 429 ---------- 430 data : dict 431 Dictionary with key value pairs. 432 """ 433 434 self.valid_prefixes([None]) 435 self.history.append("DELETE_FROM") 436 if self._db and not table_name in self._db.get_table_names(): 437 raise QueryError(f"Can not perform DELETE FROM on a non-existing table: {table_name!r}") 438 self.table = table_name 439 self.sql = f"DELETE FROM {table_name} " 440 return(self) 441 442 443 def run(self, db=None, raw = False, verbose=False) -> list[DatabaseEntry]: 444 """ 445 Execute the query in the attached database or in a seperate one. Returns the results in a table (generator of DatabaseEntry) or `None` if no results. 446 447 Parameters 448 ---------- 449 db : Database, optional 450 The database to execute to query on. 451 raw : bool, optional 452 If True: returns the raw table (list of tuples) instead of the normal table. 453 verbose : bool, optional 454 Be verbose about it. 455 """ 456 457 if not db: 458 db = self._db 459 460 if not db: 461 raise DatabaseError("Query does not have a database to execute") 462 463 try: 464 db.cursor.execute(self.sql) 465 except sqlite3.OperationalError as e: 466 raise QueryError(f"\n\n{e}\n\nError while running following sql: {self.sql}") 467 468 if verbose or self.verbose or db.verbose: 469 print(f"Executed sql: {self.sql}") 470 471 results = db.cursor.fetchall() 472 473 if raw: 474 return(results) 475 476 if self.fields == "*": 477 self.fields = db.get_column_names(self.table) 478 479 return(raw_table_to_table(results, self.fields, self.table)) 480 481 def __repr__(self) -> str: 482 return(f"> {self.sql.strip()} <")
A class for writing sql queries. Queries can be run on the attached database or a seperate one with the run
method.
Parameters
db : Database, optional
The attached Database. This is the default database to run queries on.
verbose : bool, optional
Print what is going on in the Query
226 def __init__(self, db=None, verbose=False) -> None: 227 228 self._db: Database = db 229 """The attached Database""" 230 231 self.sql = "" 232 """The current raw sql command""" 233 234 self.history = [] 235 """The history of commandmethods run on this object""" 236 237 self.fields = None 238 """The selected fields""" 239 240 self.table = None 241 """The table the sql query is interacting with""" 242 243 self.verbose = verbose 244 """Print what is going on in the `Query`"""
246 def valid_prefixes(self, prefixes: list) -> None: 247 """Check if a statement is valid given its prefix""" 248 249 prefix = None 250 if len(self.history) > 0: 251 prefix = self.history[-1] 252 if prefix in prefixes: 253 return(True) 254 raise QueryError(f"Query syntax incorrect or not supported. Prefix: \"{prefix}\" is not a part of the valid prefixes: {prefixes}")
Check if a statement is valid given its prefix
256 def SELECT(self, selection="*"): 257 """ 258 Sql `SELECT` statement. Must be followed by `FROM` statement. 259 260 Parameters 261 ---------- 262 selection : str/list, optional 263 Either a python list or sql list of table names. Selects all columns if not set. 264 """ 265 266 self.valid_prefixes([None]) 267 self.history.append("SELECT") 268 269 if isinstance(selection, str): 270 if selection == "*": 271 self.fields = "*" 272 else: 273 self.fields = string_to_list(selection) 274 self.sql += f"SELECT {selection} " 275 elif isinstance(selection, list): 276 self.fields = selection 277 self.sql += f"SELECT {', '.join(selection)} " 278 else: 279 raise QueryError("SELECT statement selection must be either `str` or `list`") 280 return(self)
282 def FROM(self, table_name): 283 """ 284 Sql `FROM` statement. Has to be preceded by a SELECT statement. Can be followed by `WHERE` statement. 285 286 Parameters 287 ---------- 288 table_name : str 289 Name of the table you are selecting from. 290 """ 291 292 self.valid_prefixes(["SELECT"]) 293 self.table = table_name 294 295 if self._db: 296 table_fields = set(self._db.get_column_names(table_name)) # check if selected fields are in table 297 298 if self.fields != "*" and self._db and not set(self.fields).issubset(table_fields): 299 raise QueryError(f"Some selected field(s): {set(self.fields) - table_fields} are not fields/columns in the table: {table_name!r}. The table has the following fields: {table_fields}") 300 301 self.history.append("FROM") 302 self.sql += f"FROM {table_name} " 303 return(self)
305 def WHERE(self, col_name:str, value = ""): 306 """ 307 Sql `WHERE` statement. Can be followed by `LIKE` statement. 308 309 Parameters 310 ---------- 311 col_name : str 312 The name of the column. You can also just pass it a statement like: `"id" = 4` instead of providing a value. 313 value : optional 314 The value of the column. 315 """ 316 317 self.valid_prefixes(["FROM", "SET", "DELETE_FROM"]) 318 self.history.append("WHERE") 319 if value != "": 320 if value == None: 321 self.sql += f"WHERE {col_name} is null" 322 else: 323 self.sql += f"WHERE {col_name} = {value_to_sql_value(value)}" 324 else: 325 self.sql += f"WHERE {col_name} " 326 if col_name.find("=") == -1: # expects LIKE statement 327 self.col = col_name.replace(" ", "") 328 return(self)
330 def LIKE(self, pattern: str): 331 """ 332 Sql LIKE statement. Has to be preceded by a WHERE statement. 333 334 Parameters 335 ---------- 336 pattern : str 337 A typical sql LIKE pattern with % and _. 338 """ 339 340 self.valid_prefixes(["WHERE"]) 341 self.history.append("LIKE") 342 self.sql += f"LIKE {value_to_sql_value(pattern)} " 343 return(self)
Sql LIKE statement. Has to be preceded by a WHERE statement.
Parameters
pattern : str A typical sql LIKE pattern with % and _.
345 def UPDATE(self, table_name: str): 346 """ 347 Sql UPDATE statement. Must be followed by `SET` statement. 348 349 Parameters 350 ---------- 351 table_name : str 352 Name of the table you are updating. 353 """ 354 355 self.valid_prefixes([None]) 356 self.history.append("UPDATE") 357 if self._db: 358 if not self._db.is_table(table_name): 359 raise QueryError(f"Database has no table called {table_name!r}") 360 self.fields = self._db.get_column_names(table_name) 361 self.table = table_name 362 self.sql += f"UPDATE {table_name} " 363 return(self)
Sql UPDATE statement. Must be followed by SET
statement.
Parameters
table_name : str Name of the table you are updating.
365 def SET(self, data: dict): 366 """ 367 Sql SET statement. Must be preceded by an UPDATE statement. Must be followed by `WHERE` statement. 368 369 Parameters 370 ---------- 371 data : dict 372 A dictionaty with key and value pairs. 373 """ 374 375 self.valid_prefixes(["UPDATE"]) 376 self.history.append("SET") 377 378 data = dict(data) 379 380 if not set(data).issubset(self.fields): 381 raise DatabaseError(f"Data keys: {set(data)} are not a subset of table fields/columns. Table fields/columns: {set(self.fields)}") 382 383 self.sql += f"SET {dict_to_sql(data)} " 384 385 return(self)
Sql SET statement. Must be preceded by an UPDATE statement. Must be followed by WHERE
statement.
Parameters
data : dict A dictionaty with key and value pairs.
387 def INSERT_INTO(self, table_name: str): 388 """ 389 Sql `INSERT INTO` statement. Must be followed by `VALUES` statement. 390 391 Parameters 392 ---------- 393 table_name : str 394 Name of the table you want to insert into. 395 """ 396 397 self.valid_prefixes([None]) 398 self.history.append("INSERT_INTO") 399 self.table = table_name 400 if self._db: 401 self.fields = self._db.get_column_names(table_name) 402 self.sql += f"INSERT INTO {table_name} " 403 return(self)
Sql INSERT INTO
statement. Must be followed by VALUES
statement.
Parameters
table_name : str Name of the table you want to insert into.
405 def VALUES(self, data: dict): 406 """ 407 Sql `VALUES` statement. Must be preceded by INSERT_INTO statement. 408 409 Parameters 410 ---------- 411 data : dict 412 Dictionary with key value pairs. 413 """ 414 415 self.valid_prefixes(["INSERT_INTO"]) 416 self.history.append("VALUES") 417 418 if not set(data).issubset(self.fields): 419 raise DatabaseError(f"Data keys: {set(data)} are not a subset of table fields/columns. Unknown keys: {set(data) - set(self.fields)}. Table fields/columns: {set(self.fields)}") 420 421 self.sql += f"({', '.join([str(v) for v in list(data)])}) VALUES ({', '.join([str(value_to_sql_value(v)) for v in data.values()])}) " 422 return(self)
Sql VALUES
statement. Must be preceded by INSERT_INTO statement.
Parameters
data : dict Dictionary with key value pairs.
424 def DELETE_FROM(self, table_name: str): 425 """ 426 Sql `DELETE FROM` statement. Must be followed by `WHERE` statement. 427 428 Parameters 429 ---------- 430 data : dict 431 Dictionary with key value pairs. 432 """ 433 434 self.valid_prefixes([None]) 435 self.history.append("DELETE_FROM") 436 if self._db and not table_name in self._db.get_table_names(): 437 raise QueryError(f"Can not perform DELETE FROM on a non-existing table: {table_name!r}") 438 self.table = table_name 439 self.sql = f"DELETE FROM {table_name} " 440 return(self)
Sql DELETE FROM
statement. Must be followed by WHERE
statement.
Parameters
data : dict Dictionary with key value pairs.
443 def run(self, db=None, raw = False, verbose=False) -> list[DatabaseEntry]: 444 """ 445 Execute the query in the attached database or in a seperate one. Returns the results in a table (generator of DatabaseEntry) or `None` if no results. 446 447 Parameters 448 ---------- 449 db : Database, optional 450 The database to execute to query on. 451 raw : bool, optional 452 If True: returns the raw table (list of tuples) instead of the normal table. 453 verbose : bool, optional 454 Be verbose about it. 455 """ 456 457 if not db: 458 db = self._db 459 460 if not db: 461 raise DatabaseError("Query does not have a database to execute") 462 463 try: 464 db.cursor.execute(self.sql) 465 except sqlite3.OperationalError as e: 466 raise QueryError(f"\n\n{e}\n\nError while running following sql: {self.sql}") 467 468 if verbose or self.verbose or db.verbose: 469 print(f"Executed sql: {self.sql}") 470 471 results = db.cursor.fetchall() 472 473 if raw: 474 return(results) 475 476 if self.fields == "*": 477 self.fields = db.get_column_names(self.table) 478 479 return(raw_table_to_table(results, self.fields, self.table))
Execute the query in the attached database or in a seperate one. Returns the results in a table (generator of DatabaseEntry) or None
if no results.
Parameters
db : Database, optional The database to execute to query on. raw : bool, optional If True: returns the raw table (list of tuples) instead of the normal table. verbose : bool, optional Be verbose about it.
487class Database: 488 """ 489 Main database class for manipulating sqlite3 databases. 490 491 Parameters 492 ---------- 493 path : str 494 Path to the database file. 495 new : bool, optional 496 A new blank database will be created where the `self.path` is pointing. 497 verbose : bool, optional 498 Enables feedback in the form of prints. 499 """ 500 501 def __init__(self, path: str, new = False, verbose=False, silent=None): 502 503 if not new and not os.path.isfile(path): 504 raise(DatabaseError(f"No database file at \"{path}\". If you want to create one, pass \"new=True\"")) 505 506 self.path = path 507 """Path to the database file.""" 508 509 self.conn = sqlite3.connect(path) 510 """The sqlite3 connection.""" 511 512 self.cursor = self.conn.cursor() 513 """The sqlite3 cursor. Use `cursor.execute(cmd)` to execute raw sql.""" 514 515 self.connected: bool = True 516 """Is true if the `Database` instance is connected to a database.""" 517 518 self.verbose=verbose 519 """Enables feedback in the form of prints.""" 520 521 self.conn.execute("PRAGMA foreign_keys = ON") 522 523 # Deprecation notice 524 if isinstance(silent, bool): 525 print("[DEPRECATION] `silent` has been removed in favor of `verbose`. The `verbose` option is `False` by default.\n") 526 527 @classmethod 528 def in_memory(cls, verbose=False): 529 """ 530 Create a database in memory. Returns the `Database` instance. 531 532 Parameters 533 ---------- 534 verbose : bool, optional 535 Enables feedback in the form of prints. 536 """ 537 return Database(":memory:", new=True, verbose=verbose) 538 539 def create_table(self, name: str, cols: list[Column]): 540 """ 541 Creates a table in the Database. 542 543 Parameters 544 ---------- 545 name : str 546 Name of the new table. 547 cols : list[Column] 548 List of columns in the new table. 549 """ 550 551 sql = f"CREATE TABLE {name} (\n" 552 553 foreign_keys: list[ForeignKey] = [] 554 555 for col in cols: 556 sql += f"{col.name!r} {col.type}" 557 558 if col.primary_key: 559 sql += " PRIMARY KEY" 560 if col.not_null: 561 sql += " NOT NULL" 562 if col.default_value: 563 sql += f" DEFAULT {col.default_value!r}" 564 if col.foreign_key: 565 foreign_keys.append(col.foreign_key) 566 sql += ",\n" 567 568 for key in foreign_keys: 569 sql += f"FOREIGN KEY({key.from_col}) REFERENCES {key.table}({key.to_col}),\n" 570 571 if key.on_update: 572 sql = sql[:-2] + f"\nON UPDATE {key.on_update},\n" 573 574 if key.on_delete: 575 sql = sql[:-2] + f"\nON DELETE {key.on_delete},\n" 576 577 578 sql = sql[:-2] + "\n)" # remove last ",\n" 579 580 self.cursor.execute(sql) 581 582 def rename_table(self, current_name: str, new_name: str): 583 """ 584 Renames a table in the database. 585 586 Parameters 587 ---------- 588 current_name : str 589 Current name of a table. 590 new_name : str 591 New name of the table. 592 """ 593 self.cursor.execute(f"ALTER TABLE {current_name} RENAME TO {new_name}") 594 595 596 def delete_table(self, table_name: str) -> None: 597 """ 598 Deletes a table in the database. 599 600 Parameters 601 ---------- 602 table_name : Name of the table. 603 """ 604 self.cursor.execute(f"DROP TABLE {table_name}") 605 606 def add_column(self, table_name: str, col: Column): 607 """ 608 Add column to a table in the database. 609 610 Parameters 611 ---------- 612 table_name : str 613 Name of the table. 614 col : Column 615 The column to add to table. 616 """ 617 618 # Check that the table exists 619 if not self.is_table(table_name): 620 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 621 622 sql = f"ALTER TABLE {table_name} ADD COLUMN {col.name} {col.type}" 623 624 if col.primary_key: 625 sql += " PRIMARY KEY" 626 if col.not_null: 627 sql += " NOT NULL" 628 if col.default_value: 629 sql += f" DEFAULT {col.default_value}" 630 if col.foreign_key: 631 raise DatabaseError(f"Sqlite3 and therefore sqlite-integrated, does not support adding columns with foreign key constraings to existing tables. They have to be declared with the creation of the table.") 632 633 self.cursor.execute(sql) 634 635 def rename_column(self, table_name: str, current_column_name: str, new_column_name: str): 636 """ 637 Renames a column in the database. 638 639 Parameters 640 ---------- 641 table_name : str 642 Name of the table. 643 current_column_name : str 644 Current name of a column. 645 new_column_name : str 646 New name of the column. 647 """ 648 649 # Check that the table exists 650 if not self.is_table(table_name): 651 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 652 653 self.cursor.execute(f"ALTER TABLE {table_name} RENAME COLUMN {current_column_name} TO {new_column_name}") 654 655 def delete_column(self, table_name: str, col): 656 """ 657 Deletes a column in a table. 658 659 Parameters 660 ---------- 661 table_name : str 662 Name of the table the column is in. 663 col : str/Column 664 Column, or column name, of the column that should be deleted. 665 """ 666 667 # Check that the table exists 668 if not self.is_table(table_name): 669 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 670 671 if col is Column: 672 col = col.name 673 674 self.cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN {col}") 675 676 677 678 def get_table_names(self) -> list: 679 """Returns the names of all tables in the database.""" 680 681 res = self.conn.execute("SELECT name FROM sqlite_master WHERE type='table';") 682 names = [] 683 for name in res: 684 names.append(name[0]) 685 return(names) 686 687 def is_table(self, table_name: str) -> bool: 688 """ 689 Check if database has a table with a certain name. 690 691 Parameters 692 ---------- 693 table_name : str 694 Name to check. 695 696 """ 697 698 if table_name in self.get_table_names(): 699 return True 700 return False 701 702 def get_table_raw(self, name: str, get_only = None) -> list: 703 """ 704 Returns all entries in a table as a list of tuples. 705 706 Parameters 707 ---------- 708 name : str 709 Name of the table. 710 get_only : list, optional 711 Can be set to a list of column/field names, to only retrieve those columns/fields. 712 """ 713 714 selected = "*" 715 716 if get_only: 717 if isinstance(get_only, list): 718 fields = self.get_column_names(name) 719 for field in get_only: 720 if not field in fields: 721 raise DatabaseError(f"Table \"{name}\" contains no field/column with the name: \"{field}\". Available fields are: {fields}") 722 selected = ','.join(get_only) 723 else: 724 raise ValueError(f"get_only can either be `None` or `list`. Got: {get_only}") 725 726 self.cursor.execute(f"SELECT {selected} FROM {name}") 727 return(self.cursor.fetchall()) 728 729 def get_table(self, name: str, get_only=None) -> list: 730 """ 731 Returns all entries in a table as a table (generator of DatabaseEntry). This function loops over all entries in the table, so it is not the best in very big databases. 732 733 Parameters 734 ---------- 735 name : str 736 Name of the table. 737 get_only : list/None, optional 738 Can be set to a list of column/field names, to only retrieve those columns/fields. 739 """ 740 741 raw_table = self.get_table_raw(name, get_only) 742 743 return(raw_table_to_table(raw_table, self.get_column_names(name), name)) 744 745 746 def get_table_cols(self, name: str) -> list[Column]: 747 """ 748 Returns a list of Column objects, that contain information about the table columns. 749 750 Parameters 751 ---------- 752 name : str 753 Name of the table. 754 """ 755 756 self.cursor.execute(f"PRAGMA table_info({name});") 757 cols_raw_info = self.cursor.fetchall() 758 759 cols = [] 760 for col_raw_info in cols_raw_info: 761 is_primary = False 762 if col_raw_info[5] == 1: 763 is_primary = True 764 not_null = False 765 if col_raw_info[3] == 1: 766 not_null = True 767 cols.append(Column(col_raw_info[1], col_raw_info[2], not_null, col_raw_info[4], is_primary, col_id=col_raw_info[0])) 768 769 770 # Add foreign keys to cols 771 self.cursor.execute(f"PRAGMA foreign_key_list({name});") 772 foreign_key_list = self.cursor.fetchall() 773 774 if len(foreign_key_list) > 0: 775 for raw_foreign_key in foreign_key_list: 776 foreign_key = ForeignKey( 777 raw_foreign_key[2], 778 raw_foreign_key[4], 779 id=raw_foreign_key[0], 780 seq=raw_foreign_key[1], 781 from_col=raw_foreign_key[3], 782 on_update=raw_foreign_key[5], 783 on_delete=raw_foreign_key[6], 784 match=raw_foreign_key[7] 785 ) 786 787 for n, col in enumerate(cols): 788 if col.name == foreign_key.from_col: 789 cols[n].foreign_key = foreign_key 790 break 791 return(cols) 792 793 def get_table_id_field(self, table: str, do_error=False) -> str: 794 """ 795 Takes a table and returns the name of the field/column marked as a `PRIMARY KEY`. (This function assumes that there is only ONE field marked as a `PRIMARY KEY`). 796 797 Parameters 798 ---------- 799 table : str 800 Name of the table. 801 do_error : bool, optional 802 If True: Raises error if the table does not contain a field marked as `PRIMARY KEY`. 803 """ 804 805 cols = self.get_table_cols(table) 806 807 for col in cols: 808 if col.primary_key == True: # col_info[5] is 1 if field is a primary key. Otherwise it is 0. 809 return col.name # col_info[1] is the name of the column 810 if do_error: 811 raise DatabaseError(f"The table `{table}` has no id_field (column defined as a `PRIMARY KEY`)") 812 return(None) 813 814 def table_overview(self, name: str, max_len:int = 40, get_only = None) -> None: 815 """ 816 Prints a pretty table (with a name). 817 818 Parameters 819 ---------- 820 name : str 821 Name of the table. 822 max_len : int, optional 823 The max number of rows shown. 824 get_only : list, optional 825 If given a list of column/field names: only shows those. 826 827 """ 828 829 text = "" # the output text 830 831 raw_table = self.get_table_raw(name, get_only=get_only) 832 833 if get_only: 834 fields = get_only 835 else: 836 fields = self.get_column_names(name) 837 838 cols = len(fields) 839 840 longest_words = [0] * cols 841 842 words_table = raw_table + [fields] 843 844 845 for col in range(cols): 846 for entry in words_table: 847 if len(str(entry[col])) > longest_words[col]: 848 longest_words[col] = len(str(entry[col])) 849 850 seperator = " ║ " 851 852 def formatRow(row, longest_words): 853 formatted_list = [] 854 for i, string in enumerate(row): 855 string = str(string) 856 formatted_list.append(string + " " * (longest_words[i] - len(string))) 857 return(seperator.join(formatted_list)) 858 859 text += formatRow(fields, longest_words) + "\n" 860 underline = "═" * (sum(longest_words) + len(seperator)) 861 862 # This block is for placing the intersections 863 offset = 0 864 for n in longest_words[:-1]: # we dont create the an intersection after the last column 865 offset += n 866 underline = underline[:offset +1] + "╬" + underline[offset:] 867 offset += len(seperator) 868 869 text += underline + "\n" 870 871 if len(raw_table) >= max_len: 872 for row in raw_table[:max_len-5]: 873 text += formatRow(row, longest_words) + "\n" 874 text += " .\n .\n .\n" 875 for row in raw_table[-5:]: 876 text += formatRow(row, longest_words) + "\n" 877 else: 878 for row in raw_table: 879 text += formatRow(row, longest_words) + "\n" 880 881 print(text) 882 883 def overview(self, more=False) -> None: 884 """ 885 Prints an overview of all the tables in the database with their fields. 886 887 Parameters 888 ---------- 889 more : optional 890 If true: Prints more information on the columns in each table. 891 """ 892 893 table_names = self.get_table_names() 894 895 # if there are no tables in database 896 if len(table_names) == 0: 897 print(f"There are no tables in sqlite database at \"{self.path}\".") 898 return(None) 899 900 text = "Tables\n" 901 for table_name in table_names: 902 text += "\t" + table_name + "\n" 903 for col in self.get_table_cols(table_name): 904 text += f"\t\t{col.name}" 905 if more: 906 text += f"\t\t[{col}]" 907 text += "\n" 908 print(text) 909 910 911 def get_column_names(self, table_name: str) -> list[str]: 912 """ 913 Returns the field/column names for a given table. 914 915 Parameters 916 ---------- 917 table_name : str 918 Name of the table. 919 """ 920 921 if not self.is_table(table_name): 922 raise DatabaseError(f"Can not get column names of non-existing table {table_name!r}.") 923 924 names = [] 925 926 for col in self.get_table_cols(table_name): 927 names.append(col.name) 928 return(names) 929 930 def is_column(self, table_name: str, col_name: str) -> bool: 931 """ 932 Returns True if the given column name exists in the given table. Else returns False. 933 934 Parameters 935 ---------- 936 table_name : str 937 Name of a table. 938 col_name : str 939 Name of a column that may be in the table. 940 """ 941 942 if col_name in self.get_column_names(table_name): 943 return(True) 944 return(False) 945 946 def fill_null(self, entry: DatabaseEntry) -> DatabaseEntry: 947 """ 948 Fills out any unpopulated fields in a DatabaseEntry (fields that exist in the database table but not in the entry) and returns it. 949 950 Parameters 951 ---------- 952 entry : DatabaseEntry 953 The DatabaseEntry. 954 """ 955 956 t_fields = self.get_column_names(entry.table) 957 e_fields = list(entry) 958 for f in e_fields: 959 t_fields.remove(f) 960 for null_field in t_fields: 961 entry[null_field] = None 962 return(entry) 963 964 965 def get_entry_by_id(self, table, ID) -> DatabaseEntry: 966 """ 967 Get table entry by id. 968 969 Parameters 970 ---------- 971 table : str 972 Name of the table. 973 ID : 974 The entry id. 975 """ 976 977 id_field = self.get_table_id_field(table, do_error=True) 978 979 if not self.is_table(table): 980 raise DatabaseError(f"Database contains no table with the name: \"{table}\". These are the available tables: {self.get_table_names()}") 981 982 sql = f"SELECT * FROM {table} WHERE {id_field} = {ID}" 983 984 self.cursor.execute(sql) 985 986 answer = self.cursor.fetchall() 987 988 # some checks 989 if len(answer) != 1: 990 if len(answer) > 1: 991 raise DatabaseError(f"There are more than one entry in table \"{table}\" with an id field \"{id_field}\" with the value \"{id}\": {answer}") 992 elif len(answer) == 0: 993 raise DatabaseError(f"There is no entry in table \"{table}\" with an id_field \"{id_field}\" with a value of {ID}") 994 else: 995 raise DatabaseError("Something went very wrong, please contact the package author") # this will never be run... i think 996 997 return(DatabaseEntry.from_raw_entry(answer[0], self.get_column_names(table), table)) 998 999 def add_entry(self, entry, table = None, fill_null=False, verbose=False) -> None: 1000 """ 1001 Add an entry to the database by passing a DatabaseEntry, or with a dictionary and specifying a table name. 1002 1003 Returns the id of the added DatabaseEntry in the table, or `None` if table does not contain a primary key. 1004 1005 The entry must have values for all fields in the table. You can pass `fill_null=True` to fill any remaining fields with `None`/`null`. 1006 1007 Parameters 1008 ---------- 1009 entry : DatabaseEntry/dict 1010 The entry. 1011 table : str, optional 1012 Name of the table the entry belongs to. **Needed if adding an entry with a dictionary**. 1013 fill_null : bool, optional 1014 Fill in unpopulated fields with null values. 1015 verbose : bool, optional 1016 Enable prints. 1017 """ 1018 1019 if type(entry) == dict: 1020 if not table: 1021 raise DatabaseError(f"Please provide the table that the data should be inserted in.") 1022 entry = DatabaseEntry(entry, table) 1023 1024 if not self.is_table(entry.table): 1025 raise DatabaseError(f"Database has no table with the name \"{self.table}\". Possible tablenames are: {self.get_table_names()}") 1026 1027 table_fields = self.get_column_names(entry.table) 1028 1029 id_field = self.get_table_id_field(entry.table) 1030 1031 if id_field: 1032 entry[id_field] = None 1033 1034 if fill_null: 1035 entry = self.fill_null(entry) 1036 1037 if set(entry) != set(table_fields): 1038 raise DatabaseError(f"entry fields are not the same as the table fields: {set(entry)} != {set(table_fields)}") 1039 1040 self.INSERT_INTO(entry.table).VALUES(entry).run() 1041 1042 if verbose or self.verbose: 1043 print(f"added entry to table \"{entry.table}\": {entry}") 1044 1045 if not self.get_table_id_field(table): 1046 return None 1047 1048 self.cursor.execute("SELECT last_insert_rowid()") 1049 return (self.cursor.fetchall()[0][0]) 1050 1051 1052 def update_entry(self, entry: dict, table=None, part=False, fill_null=False, verbose=False) -> None: 1053 """ 1054 Update entry in database with a DatabaseEntry, or with a dictionary + the name of the table you want to update. 1055 1056 Parameters 1057 ---------- 1058 entry : DatabaseEntry/dict 1059 DatabaseEntry or dictionary, if dictionary you also need to provide table and id_field. 1060 table : str, optional 1061 The table name. **Needed if updating an entry with a dictionary**. 1062 part : bool, optional 1063 If True: Only updates the provided fields. 1064 fill_null : bool, optional 1065 Fill in unpopulated fields with null values. 1066 verbose : bool, optional 1067 Enable prints. 1068 """ 1069 1070 if not isinstance(entry, DatabaseEntry): # the input is a dict 1071 if not table: 1072 raise DatabaseError(f"Please provide a table when updating an entry with a python dictionary") 1073 entry = DatabaseEntry(entry, table) 1074 1075 id_field = self.get_table_id_field(entry.table) 1076 1077 if not self.is_table(entry.table): 1078 raise DatabaseError(f"Database has no table with the name \"{entry.table}\". Possible tablenames are: {self.get_table_names()}") 1079 1080 if fill_null: 1081 entry = self.fill_null(entry) 1082 1083 # check that entry fields and table fields match 1084 table_fields = self.get_column_names(entry.table) 1085 if set(table_fields) != set(entry): 1086 if not (part and set(entry).issubset(set(table_fields))): 1087 raise DatabaseError(f"Table fields do not match entry fields: {table_fields} != {list(entry)}. Pass `part = True` or `fill_null = True` if entry are a subset of the table fields") 1088 1089 self.UPDATE(entry.table).SET(entry).WHERE(id_field, entry[id_field]).run() 1090 1091 if verbose or self.verbose: 1092 print(f"updated entry in table \"{entry.table}\": {entry}") 1093 1094 def delete_entry(self, entry: DatabaseEntry): 1095 """ 1096 Delete an entry from the database. 1097 1098 Parameters 1099 ---------- 1100 entry : DatabaseEntry 1101 The entry that is to be deleted. 1102 """ 1103 1104 id_field = self.get_table_id_field(entry.table) 1105 self.DELETE_FROM(entry.table).WHERE(id_field, entry[id_field]).run() 1106 1107 1108 def delete_entry_by_id(self, table: str, id: int): 1109 """ 1110 Deletes an entry with a certain id. (Note: the table must have a primary key column, as that is what is meant by id. It is assumed that there is only one primary key column in the table.} 1111 1112 Parameters 1113 ---------- 1114 table : str 1115 The table to delete the entry from. 1116 id : int 1117 1118 """ 1119 1120 id_field = self.get_table_id_field(table) 1121 self.DELETE_FROM(table).WHERE(id_field, id).run() 1122 1123 def save(self) -> None: 1124 """Writes any changes to the database file""" 1125 1126 self.conn.commit() 1127 1128 def close(self) -> None: 1129 """Saves and closes the database. If you want to explicitly close without saving use: `self.conn.close()`""" 1130 1131 self.conn.commit() 1132 self.conn.close() 1133 self.connected = False 1134 1135 def reconnect(self) -> None: 1136 """Reopen database after closing it""" 1137 1138 self.conn = sqlite3.connect(self.path) 1139 self.cursor = self.conn.cursor() 1140 self.connected = True 1141 1142 def delete_table(self, table_name) -> None: 1143 """ 1144 Takes a table name and deletes the table from the database. 1145 1146 Parameters 1147 ---------- 1148 table_name : str 1149 Name of the table. 1150 """ 1151 1152 self.cursor.execute(f"DROP TABLE {table_name};") 1153 1154 def table_to_dataframe(self, table) -> pd.DataFrame: 1155 """ 1156 Converts a table to a pandas.Dataframe. 1157 1158 Parameters 1159 ---------- 1160 table : str 1161 Name of the table. 1162 """ 1163 1164 cols = {} 1165 fields = self.get_column_names(table) 1166 1167 for f in fields: 1168 cols[f] = [] 1169 1170 for raw_entry in self.get_table_raw(table): 1171 for n, field in enumerate(fields): 1172 cols[field].append(raw_entry[n]) 1173 1174 return(pd.DataFrame(cols)) 1175 1176 1177 def export_to_csv(self, out_dir: str, tables: list = None, sep: str = "\t") -> None: 1178 """ 1179 Export all or some tables in the database to csv files 1180 1181 Parameters 1182 ---------- 1183 out_dir : str 1184 Path to the output directory. 1185 tables : list[str]/None, optional 1186 Can be set to only export certain tables. 1187 sep : str, optional 1188 Seperator to use when writing csv-file. 1189 """ 1190 1191 if not os.path.isdir(out_dir): 1192 raise NotADirectoryError(f"{out_dir!r} is not a directory") 1193 1194 if not tables: 1195 tables = self.get_table_names() 1196 1197 for table_name in tables: 1198 df = self.table_to_dataframe(table_name) 1199 df.to_csv(f"{out_dir}/{table_name}.csv", index=False, sep=sep) 1200 1201 def run_raw_sql(self, sql: str, verbose=False): 1202 """ 1203 Run SQL-string on the database. This returns a raw table as list of tuples. 1204 1205 Parameters 1206 ---------- 1207 sql : str 1208 SQL-string to be execured as an SQL command. 1209 verbose : bool, optional 1210 Prints the SQL-query if true 1211 """ 1212 1213 try: 1214 self.cursor.execute(sql) 1215 except sqlite3.OperationalError as e: 1216 raise QueryError(f"\n\n{e}\n\nError while running following sql: {self.sql}") 1217 1218 if verbose or self.verbose: 1219 print(f"Executed sql: {self.sql}") 1220 1221 return(self.cursor.fetchall()) 1222 1223 def SELECT(self, pattern="*") -> Query: 1224 """ 1225 Start sql SELECT query from the database. Returns a Query to build from. 1226 1227 Parameters 1228 ---------- 1229 pattern : str, optional 1230 Either a python list or sql list of table names. 1231 """ 1232 1233 return(Query(db=self).SELECT(pattern)) 1234 1235 def UPDATE(self, table_name) -> Query: 1236 """ 1237 Start sql UPDATE query from the database. Returns a Query to build from. 1238 1239 Parameters 1240 ---------- 1241 table_name : str 1242 Name of the table. 1243 """ 1244 return(Query(db=self).UPDATE(table_name)) 1245 1246 def INSERT_INTO(self, table_name) -> Query: 1247 """ 1248 Start sql INSERT INTO query from the database. Returns a Query to build from. 1249 1250 Parameters 1251 ---------- 1252 table_name : str 1253 Name of the table to insert into. 1254 """ 1255 1256 return(Query(db=self).INSERT_INTO(table_name)) 1257 1258 def DELETE_FROM(self, table_name: str) -> Query: 1259 """ 1260 Start sql DELETE FROM query from the database. Returns a Query to build from. 1261 1262 Parameters 1263 ---------- 1264 table_name : str 1265 Name of the table to delete from. 1266 """ 1267 return(Query(db=self).DELETE_FROM(table_name)) 1268 1269 1270 def __eq__(self, other: object) -> bool: 1271 tables = self.get_table_names() 1272 if tables != other.get_table_names(): 1273 return(False) 1274 1275 for table in tables: 1276 if self.get_table_raw(table) != other.get_table_raw(table): 1277 return(False) 1278 elif self.get_table_cols(table) != other.get_table_cols(table): 1279 return(False) 1280 return(True)
Main database class for manipulating sqlite3 databases.
Parameters
path : str
Path to the database file.
new : bool, optional
A new blank database will be created where the self.path
is pointing.
verbose : bool, optional
Enables feedback in the form of prints.
501 def __init__(self, path: str, new = False, verbose=False, silent=None): 502 503 if not new and not os.path.isfile(path): 504 raise(DatabaseError(f"No database file at \"{path}\". If you want to create one, pass \"new=True\"")) 505 506 self.path = path 507 """Path to the database file.""" 508 509 self.conn = sqlite3.connect(path) 510 """The sqlite3 connection.""" 511 512 self.cursor = self.conn.cursor() 513 """The sqlite3 cursor. Use `cursor.execute(cmd)` to execute raw sql.""" 514 515 self.connected: bool = True 516 """Is true if the `Database` instance is connected to a database.""" 517 518 self.verbose=verbose 519 """Enables feedback in the form of prints.""" 520 521 self.conn.execute("PRAGMA foreign_keys = ON") 522 523 # Deprecation notice 524 if isinstance(silent, bool): 525 print("[DEPRECATION] `silent` has been removed in favor of `verbose`. The `verbose` option is `False` by default.\n")
527 @classmethod 528 def in_memory(cls, verbose=False): 529 """ 530 Create a database in memory. Returns the `Database` instance. 531 532 Parameters 533 ---------- 534 verbose : bool, optional 535 Enables feedback in the form of prints. 536 """ 537 return Database(":memory:", new=True, verbose=verbose)
Create a database in memory. Returns the Database
instance.
Parameters
verbose : bool, optional Enables feedback in the form of prints.
539 def create_table(self, name: str, cols: list[Column]): 540 """ 541 Creates a table in the Database. 542 543 Parameters 544 ---------- 545 name : str 546 Name of the new table. 547 cols : list[Column] 548 List of columns in the new table. 549 """ 550 551 sql = f"CREATE TABLE {name} (\n" 552 553 foreign_keys: list[ForeignKey] = [] 554 555 for col in cols: 556 sql += f"{col.name!r} {col.type}" 557 558 if col.primary_key: 559 sql += " PRIMARY KEY" 560 if col.not_null: 561 sql += " NOT NULL" 562 if col.default_value: 563 sql += f" DEFAULT {col.default_value!r}" 564 if col.foreign_key: 565 foreign_keys.append(col.foreign_key) 566 sql += ",\n" 567 568 for key in foreign_keys: 569 sql += f"FOREIGN KEY({key.from_col}) REFERENCES {key.table}({key.to_col}),\n" 570 571 if key.on_update: 572 sql = sql[:-2] + f"\nON UPDATE {key.on_update},\n" 573 574 if key.on_delete: 575 sql = sql[:-2] + f"\nON DELETE {key.on_delete},\n" 576 577 578 sql = sql[:-2] + "\n)" # remove last ",\n" 579 580 self.cursor.execute(sql)
Creates a table in the Database.
Parameters
name : str Name of the new table. cols : list[Column] List of columns in the new table.
582 def rename_table(self, current_name: str, new_name: str): 583 """ 584 Renames a table in the database. 585 586 Parameters 587 ---------- 588 current_name : str 589 Current name of a table. 590 new_name : str 591 New name of the table. 592 """ 593 self.cursor.execute(f"ALTER TABLE {current_name} RENAME TO {new_name}")
Renames a table in the database.
Parameters
current_name : str Current name of a table. new_name : str New name of the table.
1142 def delete_table(self, table_name) -> None: 1143 """ 1144 Takes a table name and deletes the table from the database. 1145 1146 Parameters 1147 ---------- 1148 table_name : str 1149 Name of the table. 1150 """ 1151 1152 self.cursor.execute(f"DROP TABLE {table_name};")
Takes a table name and deletes the table from the database.
Parameters
table_name : str Name of the table.
606 def add_column(self, table_name: str, col: Column): 607 """ 608 Add column to a table in the database. 609 610 Parameters 611 ---------- 612 table_name : str 613 Name of the table. 614 col : Column 615 The column to add to table. 616 """ 617 618 # Check that the table exists 619 if not self.is_table(table_name): 620 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 621 622 sql = f"ALTER TABLE {table_name} ADD COLUMN {col.name} {col.type}" 623 624 if col.primary_key: 625 sql += " PRIMARY KEY" 626 if col.not_null: 627 sql += " NOT NULL" 628 if col.default_value: 629 sql += f" DEFAULT {col.default_value}" 630 if col.foreign_key: 631 raise DatabaseError(f"Sqlite3 and therefore sqlite-integrated, does not support adding columns with foreign key constraings to existing tables. They have to be declared with the creation of the table.") 632 633 self.cursor.execute(sql)
Add column to a table in the database.
Parameters
table_name : str Name of the table. col : Column The column to add to table.
635 def rename_column(self, table_name: str, current_column_name: str, new_column_name: str): 636 """ 637 Renames a column in the database. 638 639 Parameters 640 ---------- 641 table_name : str 642 Name of the table. 643 current_column_name : str 644 Current name of a column. 645 new_column_name : str 646 New name of the column. 647 """ 648 649 # Check that the table exists 650 if not self.is_table(table_name): 651 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 652 653 self.cursor.execute(f"ALTER TABLE {table_name} RENAME COLUMN {current_column_name} TO {new_column_name}")
Renames a column in the database.
Parameters
table_name : str Name of the table. current_column_name : str Current name of a column. new_column_name : str New name of the column.
655 def delete_column(self, table_name: str, col): 656 """ 657 Deletes a column in a table. 658 659 Parameters 660 ---------- 661 table_name : str 662 Name of the table the column is in. 663 col : str/Column 664 Column, or column name, of the column that should be deleted. 665 """ 666 667 # Check that the table exists 668 if not self.is_table(table_name): 669 raise DatabaseError(f"Database contains no table with the name {table_name!r}") 670 671 if col is Column: 672 col = col.name 673 674 self.cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN {col}")
Deletes a column in a table.
Parameters
table_name : str Name of the table the column is in. col : str/Column Column, or column name, of the column that should be deleted.
678 def get_table_names(self) -> list: 679 """Returns the names of all tables in the database.""" 680 681 res = self.conn.execute("SELECT name FROM sqlite_master WHERE type='table';") 682 names = [] 683 for name in res: 684 names.append(name[0]) 685 return(names)
Returns the names of all tables in the database.
687 def is_table(self, table_name: str) -> bool: 688 """ 689 Check if database has a table with a certain name. 690 691 Parameters 692 ---------- 693 table_name : str 694 Name to check. 695 696 """ 697 698 if table_name in self.get_table_names(): 699 return True 700 return False
Check if database has a table with a certain name.
Parameters
table_name : str Name to check.
702 def get_table_raw(self, name: str, get_only = None) -> list: 703 """ 704 Returns all entries in a table as a list of tuples. 705 706 Parameters 707 ---------- 708 name : str 709 Name of the table. 710 get_only : list, optional 711 Can be set to a list of column/field names, to only retrieve those columns/fields. 712 """ 713 714 selected = "*" 715 716 if get_only: 717 if isinstance(get_only, list): 718 fields = self.get_column_names(name) 719 for field in get_only: 720 if not field in fields: 721 raise DatabaseError(f"Table \"{name}\" contains no field/column with the name: \"{field}\". Available fields are: {fields}") 722 selected = ','.join(get_only) 723 else: 724 raise ValueError(f"get_only can either be `None` or `list`. Got: {get_only}") 725 726 self.cursor.execute(f"SELECT {selected} FROM {name}") 727 return(self.cursor.fetchall())
Returns all entries in a table as a list of tuples.
Parameters
name : str Name of the table. get_only : list, optional Can be set to a list of column/field names, to only retrieve those columns/fields.
729 def get_table(self, name: str, get_only=None) -> list: 730 """ 731 Returns all entries in a table as a table (generator of DatabaseEntry). This function loops over all entries in the table, so it is not the best in very big databases. 732 733 Parameters 734 ---------- 735 name : str 736 Name of the table. 737 get_only : list/None, optional 738 Can be set to a list of column/field names, to only retrieve those columns/fields. 739 """ 740 741 raw_table = self.get_table_raw(name, get_only) 742 743 return(raw_table_to_table(raw_table, self.get_column_names(name), name))
Returns all entries in a table as a table (generator of DatabaseEntry). This function loops over all entries in the table, so it is not the best in very big databases.
Parameters
name : str Name of the table. get_only : list/None, optional Can be set to a list of column/field names, to only retrieve those columns/fields.
746 def get_table_cols(self, name: str) -> list[Column]: 747 """ 748 Returns a list of Column objects, that contain information about the table columns. 749 750 Parameters 751 ---------- 752 name : str 753 Name of the table. 754 """ 755 756 self.cursor.execute(f"PRAGMA table_info({name});") 757 cols_raw_info = self.cursor.fetchall() 758 759 cols = [] 760 for col_raw_info in cols_raw_info: 761 is_primary = False 762 if col_raw_info[5] == 1: 763 is_primary = True 764 not_null = False 765 if col_raw_info[3] == 1: 766 not_null = True 767 cols.append(Column(col_raw_info[1], col_raw_info[2], not_null, col_raw_info[4], is_primary, col_id=col_raw_info[0])) 768 769 770 # Add foreign keys to cols 771 self.cursor.execute(f"PRAGMA foreign_key_list({name});") 772 foreign_key_list = self.cursor.fetchall() 773 774 if len(foreign_key_list) > 0: 775 for raw_foreign_key in foreign_key_list: 776 foreign_key = ForeignKey( 777 raw_foreign_key[2], 778 raw_foreign_key[4], 779 id=raw_foreign_key[0], 780 seq=raw_foreign_key[1], 781 from_col=raw_foreign_key[3], 782 on_update=raw_foreign_key[5], 783 on_delete=raw_foreign_key[6], 784 match=raw_foreign_key[7] 785 ) 786 787 for n, col in enumerate(cols): 788 if col.name == foreign_key.from_col: 789 cols[n].foreign_key = foreign_key 790 break 791 return(cols)
Returns a list of Column objects, that contain information about the table columns.
Parameters
name : str Name of the table.
793 def get_table_id_field(self, table: str, do_error=False) -> str: 794 """ 795 Takes a table and returns the name of the field/column marked as a `PRIMARY KEY`. (This function assumes that there is only ONE field marked as a `PRIMARY KEY`). 796 797 Parameters 798 ---------- 799 table : str 800 Name of the table. 801 do_error : bool, optional 802 If True: Raises error if the table does not contain a field marked as `PRIMARY KEY`. 803 """ 804 805 cols = self.get_table_cols(table) 806 807 for col in cols: 808 if col.primary_key == True: # col_info[5] is 1 if field is a primary key. Otherwise it is 0. 809 return col.name # col_info[1] is the name of the column 810 if do_error: 811 raise DatabaseError(f"The table `{table}` has no id_field (column defined as a `PRIMARY KEY`)") 812 return(None)
Takes a table and returns the name of the field/column marked as a PRIMARY KEY
. (This function assumes that there is only ONE field marked as a PRIMARY KEY
).
Parameters
table : str
Name of the table.
do_error : bool, optional
If True: Raises error if the table does not contain a field marked as PRIMARY KEY
.
814 def table_overview(self, name: str, max_len:int = 40, get_only = None) -> None: 815 """ 816 Prints a pretty table (with a name). 817 818 Parameters 819 ---------- 820 name : str 821 Name of the table. 822 max_len : int, optional 823 The max number of rows shown. 824 get_only : list, optional 825 If given a list of column/field names: only shows those. 826 827 """ 828 829 text = "" # the output text 830 831 raw_table = self.get_table_raw(name, get_only=get_only) 832 833 if get_only: 834 fields = get_only 835 else: 836 fields = self.get_column_names(name) 837 838 cols = len(fields) 839 840 longest_words = [0] * cols 841 842 words_table = raw_table + [fields] 843 844 845 for col in range(cols): 846 for entry in words_table: 847 if len(str(entry[col])) > longest_words[col]: 848 longest_words[col] = len(str(entry[col])) 849 850 seperator = " ║ " 851 852 def formatRow(row, longest_words): 853 formatted_list = [] 854 for i, string in enumerate(row): 855 string = str(string) 856 formatted_list.append(string + " " * (longest_words[i] - len(string))) 857 return(seperator.join(formatted_list)) 858 859 text += formatRow(fields, longest_words) + "\n" 860 underline = "═" * (sum(longest_words) + len(seperator)) 861 862 # This block is for placing the intersections 863 offset = 0 864 for n in longest_words[:-1]: # we dont create the an intersection after the last column 865 offset += n 866 underline = underline[:offset +1] + "╬" + underline[offset:] 867 offset += len(seperator) 868 869 text += underline + "\n" 870 871 if len(raw_table) >= max_len: 872 for row in raw_table[:max_len-5]: 873 text += formatRow(row, longest_words) + "\n" 874 text += " .\n .\n .\n" 875 for row in raw_table[-5:]: 876 text += formatRow(row, longest_words) + "\n" 877 else: 878 for row in raw_table: 879 text += formatRow(row, longest_words) + "\n" 880 881 print(text)
Prints a pretty table (with a name).
Parameters
name : str Name of the table. max_len : int, optional The max number of rows shown. get_only : list, optional If given a list of column/field names: only shows those.
883 def overview(self, more=False) -> None: 884 """ 885 Prints an overview of all the tables in the database with their fields. 886 887 Parameters 888 ---------- 889 more : optional 890 If true: Prints more information on the columns in each table. 891 """ 892 893 table_names = self.get_table_names() 894 895 # if there are no tables in database 896 if len(table_names) == 0: 897 print(f"There are no tables in sqlite database at \"{self.path}\".") 898 return(None) 899 900 text = "Tables\n" 901 for table_name in table_names: 902 text += "\t" + table_name + "\n" 903 for col in self.get_table_cols(table_name): 904 text += f"\t\t{col.name}" 905 if more: 906 text += f"\t\t[{col}]" 907 text += "\n" 908 print(text)
Prints an overview of all the tables in the database with their fields.
Parameters
more : optional If true: Prints more information on the columns in each table.
911 def get_column_names(self, table_name: str) -> list[str]: 912 """ 913 Returns the field/column names for a given table. 914 915 Parameters 916 ---------- 917 table_name : str 918 Name of the table. 919 """ 920 921 if not self.is_table(table_name): 922 raise DatabaseError(f"Can not get column names of non-existing table {table_name!r}.") 923 924 names = [] 925 926 for col in self.get_table_cols(table_name): 927 names.append(col.name) 928 return(names)
Returns the field/column names for a given table.
Parameters
table_name : str Name of the table.
930 def is_column(self, table_name: str, col_name: str) -> bool: 931 """ 932 Returns True if the given column name exists in the given table. Else returns False. 933 934 Parameters 935 ---------- 936 table_name : str 937 Name of a table. 938 col_name : str 939 Name of a column that may be in the table. 940 """ 941 942 if col_name in self.get_column_names(table_name): 943 return(True) 944 return(False)
Returns True if the given column name exists in the given table. Else returns False.
Parameters
table_name : str Name of a table. col_name : str Name of a column that may be in the table.
946 def fill_null(self, entry: DatabaseEntry) -> DatabaseEntry: 947 """ 948 Fills out any unpopulated fields in a DatabaseEntry (fields that exist in the database table but not in the entry) and returns it. 949 950 Parameters 951 ---------- 952 entry : DatabaseEntry 953 The DatabaseEntry. 954 """ 955 956 t_fields = self.get_column_names(entry.table) 957 e_fields = list(entry) 958 for f in e_fields: 959 t_fields.remove(f) 960 for null_field in t_fields: 961 entry[null_field] = None 962 return(entry)
Fills out any unpopulated fields in a DatabaseEntry (fields that exist in the database table but not in the entry) and returns it.
Parameters
entry : DatabaseEntry The DatabaseEntry.
965 def get_entry_by_id(self, table, ID) -> DatabaseEntry: 966 """ 967 Get table entry by id. 968 969 Parameters 970 ---------- 971 table : str 972 Name of the table. 973 ID : 974 The entry id. 975 """ 976 977 id_field = self.get_table_id_field(table, do_error=True) 978 979 if not self.is_table(table): 980 raise DatabaseError(f"Database contains no table with the name: \"{table}\". These are the available tables: {self.get_table_names()}") 981 982 sql = f"SELECT * FROM {table} WHERE {id_field} = {ID}" 983 984 self.cursor.execute(sql) 985 986 answer = self.cursor.fetchall() 987 988 # some checks 989 if len(answer) != 1: 990 if len(answer) > 1: 991 raise DatabaseError(f"There are more than one entry in table \"{table}\" with an id field \"{id_field}\" with the value \"{id}\": {answer}") 992 elif len(answer) == 0: 993 raise DatabaseError(f"There is no entry in table \"{table}\" with an id_field \"{id_field}\" with a value of {ID}") 994 else: 995 raise DatabaseError("Something went very wrong, please contact the package author") # this will never be run... i think 996 997 return(DatabaseEntry.from_raw_entry(answer[0], self.get_column_names(table), table))
Get table entry by id.
Parameters
table : str
Name of the table.
ID :
The entry id.
999 def add_entry(self, entry, table = None, fill_null=False, verbose=False) -> None: 1000 """ 1001 Add an entry to the database by passing a DatabaseEntry, or with a dictionary and specifying a table name. 1002 1003 Returns the id of the added DatabaseEntry in the table, or `None` if table does not contain a primary key. 1004 1005 The entry must have values for all fields in the table. You can pass `fill_null=True` to fill any remaining fields with `None`/`null`. 1006 1007 Parameters 1008 ---------- 1009 entry : DatabaseEntry/dict 1010 The entry. 1011 table : str, optional 1012 Name of the table the entry belongs to. **Needed if adding an entry with a dictionary**. 1013 fill_null : bool, optional 1014 Fill in unpopulated fields with null values. 1015 verbose : bool, optional 1016 Enable prints. 1017 """ 1018 1019 if type(entry) == dict: 1020 if not table: 1021 raise DatabaseError(f"Please provide the table that the data should be inserted in.") 1022 entry = DatabaseEntry(entry, table) 1023 1024 if not self.is_table(entry.table): 1025 raise DatabaseError(f"Database has no table with the name \"{self.table}\". Possible tablenames are: {self.get_table_names()}") 1026 1027 table_fields = self.get_column_names(entry.table) 1028 1029 id_field = self.get_table_id_field(entry.table) 1030 1031 if id_field: 1032 entry[id_field] = None 1033 1034 if fill_null: 1035 entry = self.fill_null(entry) 1036 1037 if set(entry) != set(table_fields): 1038 raise DatabaseError(f"entry fields are not the same as the table fields: {set(entry)} != {set(table_fields)}") 1039 1040 self.INSERT_INTO(entry.table).VALUES(entry).run() 1041 1042 if verbose or self.verbose: 1043 print(f"added entry to table \"{entry.table}\": {entry}") 1044 1045 if not self.get_table_id_field(table): 1046 return None 1047 1048 self.cursor.execute("SELECT last_insert_rowid()") 1049 return (self.cursor.fetchall()[0][0])
Add an entry to the database by passing a DatabaseEntry, or with a dictionary and specifying a table name.
Returns the id of the added DatabaseEntry in the table, or None
if table does not contain a primary key.
The entry must have values for all fields in the table. You can pass fill_null=True
to fill any remaining fields with None
/null
.
Parameters
entry : DatabaseEntry/dict The entry. table : str, optional Name of the table the entry belongs to. Needed if adding an entry with a dictionary. fill_null : bool, optional Fill in unpopulated fields with null values. verbose : bool, optional Enable prints.
1052 def update_entry(self, entry: dict, table=None, part=False, fill_null=False, verbose=False) -> None: 1053 """ 1054 Update entry in database with a DatabaseEntry, or with a dictionary + the name of the table you want to update. 1055 1056 Parameters 1057 ---------- 1058 entry : DatabaseEntry/dict 1059 DatabaseEntry or dictionary, if dictionary you also need to provide table and id_field. 1060 table : str, optional 1061 The table name. **Needed if updating an entry with a dictionary**. 1062 part : bool, optional 1063 If True: Only updates the provided fields. 1064 fill_null : bool, optional 1065 Fill in unpopulated fields with null values. 1066 verbose : bool, optional 1067 Enable prints. 1068 """ 1069 1070 if not isinstance(entry, DatabaseEntry): # the input is a dict 1071 if not table: 1072 raise DatabaseError(f"Please provide a table when updating an entry with a python dictionary") 1073 entry = DatabaseEntry(entry, table) 1074 1075 id_field = self.get_table_id_field(entry.table) 1076 1077 if not self.is_table(entry.table): 1078 raise DatabaseError(f"Database has no table with the name \"{entry.table}\". Possible tablenames are: {self.get_table_names()}") 1079 1080 if fill_null: 1081 entry = self.fill_null(entry) 1082 1083 # check that entry fields and table fields match 1084 table_fields = self.get_column_names(entry.table) 1085 if set(table_fields) != set(entry): 1086 if not (part and set(entry).issubset(set(table_fields))): 1087 raise DatabaseError(f"Table fields do not match entry fields: {table_fields} != {list(entry)}. Pass `part = True` or `fill_null = True` if entry are a subset of the table fields") 1088 1089 self.UPDATE(entry.table).SET(entry).WHERE(id_field, entry[id_field]).run() 1090 1091 if verbose or self.verbose: 1092 print(f"updated entry in table \"{entry.table}\": {entry}")
Update entry in database with a DatabaseEntry, or with a dictionary + the name of the table you want to update.
Parameters
entry : DatabaseEntry/dict DatabaseEntry or dictionary, if dictionary you also need to provide table and id_field. table : str, optional The table name. Needed if updating an entry with a dictionary. part : bool, optional If True: Only updates the provided fields. fill_null : bool, optional Fill in unpopulated fields with null values. verbose : bool, optional Enable prints.
1094 def delete_entry(self, entry: DatabaseEntry): 1095 """ 1096 Delete an entry from the database. 1097 1098 Parameters 1099 ---------- 1100 entry : DatabaseEntry 1101 The entry that is to be deleted. 1102 """ 1103 1104 id_field = self.get_table_id_field(entry.table) 1105 self.DELETE_FROM(entry.table).WHERE(id_field, entry[id_field]).run()
Delete an entry from the database.
Parameters
entry : DatabaseEntry The entry that is to be deleted.
1108 def delete_entry_by_id(self, table: str, id: int): 1109 """ 1110 Deletes an entry with a certain id. (Note: the table must have a primary key column, as that is what is meant by id. It is assumed that there is only one primary key column in the table.} 1111 1112 Parameters 1113 ---------- 1114 table : str 1115 The table to delete the entry from. 1116 id : int 1117 1118 """ 1119 1120 id_field = self.get_table_id_field(table) 1121 self.DELETE_FROM(table).WHERE(id_field, id).run()
Deletes an entry with a certain id. (Note: the table must have a primary key column, as that is what is meant by id. It is assumed that there is only one primary key column in the table.}
Parameters
table : str The table to delete the entry from. id : int
1123 def save(self) -> None: 1124 """Writes any changes to the database file""" 1125 1126 self.conn.commit()
Writes any changes to the database file
1128 def close(self) -> None: 1129 """Saves and closes the database. If you want to explicitly close without saving use: `self.conn.close()`""" 1130 1131 self.conn.commit() 1132 self.conn.close() 1133 self.connected = False
Saves and closes the database. If you want to explicitly close without saving use: self.conn.close()
1135 def reconnect(self) -> None: 1136 """Reopen database after closing it""" 1137 1138 self.conn = sqlite3.connect(self.path) 1139 self.cursor = self.conn.cursor() 1140 self.connected = True
Reopen database after closing it
1154 def table_to_dataframe(self, table) -> pd.DataFrame: 1155 """ 1156 Converts a table to a pandas.Dataframe. 1157 1158 Parameters 1159 ---------- 1160 table : str 1161 Name of the table. 1162 """ 1163 1164 cols = {} 1165 fields = self.get_column_names(table) 1166 1167 for f in fields: 1168 cols[f] = [] 1169 1170 for raw_entry in self.get_table_raw(table): 1171 for n, field in enumerate(fields): 1172 cols[field].append(raw_entry[n]) 1173 1174 return(pd.DataFrame(cols))
Converts a table to a pandas.Dataframe.
Parameters
table : str Name of the table.
1177 def export_to_csv(self, out_dir: str, tables: list = None, sep: str = "\t") -> None: 1178 """ 1179 Export all or some tables in the database to csv files 1180 1181 Parameters 1182 ---------- 1183 out_dir : str 1184 Path to the output directory. 1185 tables : list[str]/None, optional 1186 Can be set to only export certain tables. 1187 sep : str, optional 1188 Seperator to use when writing csv-file. 1189 """ 1190 1191 if not os.path.isdir(out_dir): 1192 raise NotADirectoryError(f"{out_dir!r} is not a directory") 1193 1194 if not tables: 1195 tables = self.get_table_names() 1196 1197 for table_name in tables: 1198 df = self.table_to_dataframe(table_name) 1199 df.to_csv(f"{out_dir}/{table_name}.csv", index=False, sep=sep)
Export all or some tables in the database to csv files
Parameters
out_dir : str Path to the output directory. tables : list[str]/None, optional Can be set to only export certain tables. sep : str, optional Seperator to use when writing csv-file.
1201 def run_raw_sql(self, sql: str, verbose=False): 1202 """ 1203 Run SQL-string on the database. This returns a raw table as list of tuples. 1204 1205 Parameters 1206 ---------- 1207 sql : str 1208 SQL-string to be execured as an SQL command. 1209 verbose : bool, optional 1210 Prints the SQL-query if true 1211 """ 1212 1213 try: 1214 self.cursor.execute(sql) 1215 except sqlite3.OperationalError as e: 1216 raise QueryError(f"\n\n{e}\n\nError while running following sql: {self.sql}") 1217 1218 if verbose or self.verbose: 1219 print(f"Executed sql: {self.sql}") 1220 1221 return(self.cursor.fetchall())
Run SQL-string on the database. This returns a raw table as list of tuples.
Parameters
sql : str SQL-string to be execured as an SQL command. verbose : bool, optional Prints the SQL-query if true
1223 def SELECT(self, pattern="*") -> Query: 1224 """ 1225 Start sql SELECT query from the database. Returns a Query to build from. 1226 1227 Parameters 1228 ---------- 1229 pattern : str, optional 1230 Either a python list or sql list of table names. 1231 """ 1232 1233 return(Query(db=self).SELECT(pattern))
Start sql SELECT query from the database. Returns a Query to build from.
Parameters
pattern : str, optional Either a python list or sql list of table names.
1235 def UPDATE(self, table_name) -> Query: 1236 """ 1237 Start sql UPDATE query from the database. Returns a Query to build from. 1238 1239 Parameters 1240 ---------- 1241 table_name : str 1242 Name of the table. 1243 """ 1244 return(Query(db=self).UPDATE(table_name))
Start sql UPDATE query from the database. Returns a Query to build from.
Parameters
table_name : str Name of the table.
1246 def INSERT_INTO(self, table_name) -> Query: 1247 """ 1248 Start sql INSERT INTO query from the database. Returns a Query to build from. 1249 1250 Parameters 1251 ---------- 1252 table_name : str 1253 Name of the table to insert into. 1254 """ 1255 1256 return(Query(db=self).INSERT_INTO(table_name))
Start sql INSERT INTO query from the database. Returns a Query to build from.
Parameters
table_name : str Name of the table to insert into.
1258 def DELETE_FROM(self, table_name: str) -> Query: 1259 """ 1260 Start sql DELETE FROM query from the database. Returns a Query to build from. 1261 1262 Parameters 1263 ---------- 1264 table_name : str 1265 Name of the table to delete from. 1266 """ 1267 return(Query(db=self).DELETE_FROM(table_name))
Start sql DELETE FROM query from the database. Returns a Query to build from.
Parameters
table_name : str Name of the table to delete from.