import copy import time import warnings from collections import deque from contextlib import contextmanager from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db import DEFAULT_DB_ALIAS from django.db.backends import utils from django.db.backends.signals import connection_created from django.db.transaction import TransactionManagementError from django.db.utils import DatabaseError, DatabaseErrorWrapper from django.utils import timezone from django.utils.functional import cached_property from django.utils.six.moves import _thread as thread try: import pytz except ImportError: pytz = None NO_DB_ALIAS = '__no_db__' class BaseDatabaseWrapper(object): """ Represents a database connection. """ # Mapping of Field objects to their column types. data_types = {} # Mapping of Field objects to their SQL suffix such as AUTOINCREMENT. data_types_suffix = {} # Mapping of Field objects to their SQL for CHECK constraints. data_type_check_constraints = {} ops = None vendor = 'unknown' SchemaEditorClass = None queries_limit = 9000 def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS, allow_thread_sharing=False): # Connection related attributes. # The underlying database connection. self.connection = None # `settings_dict` should be a dictionary containing keys such as # NAME, USER, etc. It's called `settings_dict` instead of `settings` # to disambiguate it from Django settings modules. self.settings_dict = settings_dict self.alias = alias # Query logging in debug mode or when explicitly enabled. self.queries_log = deque(maxlen=self.queries_limit) self.force_debug_cursor = False # Transaction related attributes. # Tracks if the connection is in autocommit mode. Per PEP 249, by # default, it isn't. self.autocommit = False # Tracks if the connection is in a transaction managed by 'atomic'. self.in_atomic_block = False # Increment to generate unique savepoint ids. self.savepoint_state = 0 # List of savepoints created by 'atomic'. self.savepoint_ids = [] # Tracks if the outermost 'atomic' block should commit on exit, # ie. if autocommit was active on entry. self.commit_on_exit = True # Tracks if the transaction should be rolled back to the next # available savepoint because of an exception in an inner block. self.needs_rollback = False # Connection termination related attributes. self.close_at = None self.closed_in_transaction = False self.errors_occurred = False # Thread-safety related attributes. self.allow_thread_sharing = allow_thread_sharing self._thread_ident = thread.get_ident() # A list of no-argument functions to run when the transaction commits. # Each entry is an (sids, func) tuple, where sids is a set of the # active savepoint IDs when this function was registered. self.run_on_commit = [] # Should we run the on-commit hooks the next time set_autocommit(True) # is called? self.run_commit_hooks_on_set_autocommit_on = False @cached_property def timezone(self): """ Time zone for datetimes stored as naive values in the database. Returns a tzinfo object or None. This is only needed when time zone support is enabled and the database doesn't support time zones. (When the database supports time zones, the adapter handles aware datetimes so Django doesn't need to.) """ if not settings.USE_TZ: return None elif self.features.supports_timezones: return None elif self.settings_dict['TIME_ZONE'] is None: return timezone.utc else: # Only this branch requires pytz. return pytz.timezone(self.settings_dict['TIME_ZONE']) @cached_property def timezone_name(self): """ Name of the time zone of the database connection. """ if not settings.USE_TZ: return settings.TIME_ZONE elif self.settings_dict['TIME_ZONE'] is None: return 'UTC' else: return self.settings_dict['TIME_ZONE'] @property def queries_logged(self): return self.force_debug_cursor or settings.DEBUG @property def queries(self): if len(self.queries_log) == self.queries_log.maxlen: warnings.warn( "Limit for query logging exceeded, only the last {} queries " "will be returned.".format(self.queries_log.maxlen)) return list(self.queries_log) # ##### Backend-specific methods for creating connections and cursors ##### def get_connection_params(self): """Returns a dict of parameters suitable for get_new_connection.""" raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a get_connection_params() method') def get_new_connection(self, conn_params): """Opens a connection to the database.""" raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a get_new_connection() method') def init_connection_state(self): """Initializes the database connection settings.""" raise NotImplementedError('subclasses of BaseDatabaseWrapper may require an init_connection_state() method') def create_cursor(self): """Creates a cursor. Assumes that a connection is established.""" raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a create_cursor() method') # ##### Backend-specific methods for creating connections ##### def connect(self): """Connects to the database. Assumes that the connection is closed.""" # Check for invalid configurations. self.check_settings() # In case the previous connection was closed while in an atomic block self.in_atomic_block = False self.savepoint_ids = [] self.needs_rollback = False # Reset parameters defining when to close the connection max_age = self.settings_dict['CONN_MAX_AGE'] self.close_at = None if max_age is None else time.time() + max_age self.closed_in_transaction = False self.errors_occurred = False # Establish the connection conn_params = self.get_connection_params() self.connection = self.get_new_connection(conn_params) self.set_autocommit(self.settings_dict['AUTOCOMMIT']) self.init_connection_state() connection_created.send(sender=self.__class__, connection=self) self.run_on_commit = [] def check_settings(self): if self.settings_dict['TIME_ZONE'] is not None: if not settings.USE_TZ: raise ImproperlyConfigured( "Connection '%s' cannot set TIME_ZONE because USE_TZ is " "False." % self.alias) elif self.features.supports_timezones: raise ImproperlyConfigured( "Connection '%s' cannot set TIME_ZONE because its engine " "handles time zones conversions natively." % self.alias) elif pytz is None: raise ImproperlyConfigured( "Connection '%s' cannot set TIME_ZONE because pytz isn't " "installed." % self.alias) def ensure_connection(self): """ Guarantees that a connection to the database is established. """ if self.connection is None: with self.wrap_database_errors: self.connect() # ##### Backend-specific wrappers for PEP-249 connection methods ##### def _cursor(self): self.ensure_connection() with self.wrap_database_errors: return self.create_cursor() def _commit(self): if self.connection is not None: with self.wrap_database_errors: return self.connection.commit() def _rollback(self): if self.connection is not None: with self.wrap_database_errors: return self.connection.rollback() def _close(self): if self.connection is not None: with self.wrap_database_errors: return self.connection.close() # ##### Generic wrappers for PEP-249 connection methods ##### def cursor(self): """ Creates a cursor, opening a connection if necessary. """ self.validate_thread_sharing() if self.queries_logged: cursor = self.make_debug_cursor(self._cursor()) else: cursor = self.make_cursor(self._cursor()) return cursor def commit(self): """ Commits a transaction and resets the dirty flag. """ self.validate_thread_sharing() self.validate_no_atomic_block() self._commit() # A successful commit means that the database connection works. self.errors_occurred = False self.run_commit_hooks_on_set_autocommit_on = True def rollback(self): """ Rolls back a transaction and resets the dirty flag. """ self.validate_thread_sharing() self.validate_no_atomic_block() self._rollback() # A successful rollback means that the database connection works. self.errors_occurred = False self.run_on_commit = [] def close(self): """ Closes the connection to the database. """ self.validate_thread_sharing() self.run_on_commit = [] # Don't call validate_no_atomic_block() to avoid making it difficult # to get rid of a connection in an invalid state. The next connect() # will reset the transaction state anyway. if self.closed_in_transaction or self.connection is None: return try: self._close() finally: if self.in_atomic_block: self.closed_in_transaction = True self.needs_rollback = True else: self.connection = None # ##### Backend-specific savepoint management methods ##### def _savepoint(self, sid): with self.cursor() as cursor: cursor.execute(self.ops.savepoint_create_sql(sid)) def _savepoint_rollback(self, sid): with self.cursor() as cursor: cursor.execute(self.ops.savepoint_rollback_sql(sid)) def _savepoint_commit(self, sid): with self.cursor() as cursor: cursor.execute(self.ops.savepoint_commit_sql(sid)) def _savepoint_allowed(self): # Savepoints cannot be created outside a transaction return self.features.uses_savepoints and not self.get_autocommit() # ##### Generic savepoint management methods ##### def savepoint(self): """ Creates a savepoint inside the current transaction. Returns an identifier for the savepoint that will be used for the subsequent rollback or commit. Does nothing if savepoints are not supported. """ if not self._savepoint_allowed(): return thread_ident = thread.get_ident() tid = str(thread_ident).replace('-', '') self.savepoint_state += 1 sid = "s%s_x%d" % (tid, self.savepoint_state) self.validate_thread_sharing() self._savepoint(sid) return sid def savepoint_rollback(self, sid): """ Rolls back to a savepoint. Does nothing if savepoints are not supported. """ if not self._savepoint_allowed(): return self.validate_thread_sharing() self._savepoint_rollback(sid) # Remove any callbacks registered while this savepoint was active. self.run_on_commit = [ (sids, func) for (sids, func) in self.run_on_commit if sid not in sids ] def savepoint_commit(self, sid): """ Releases a savepoint. Does nothing if savepoints are not supported. """ if not self._savepoint_allowed(): return self.validate_thread_sharing() self._savepoint_commit(sid) def clean_savepoints(self): """ Resets the counter used to generate unique savepoint ids in this thread. """ self.savepoint_state = 0 # ##### Backend-specific transaction management methods ##### def _set_autocommit(self, autocommit): """ Backend-specific implementation to enable or disable autocommit. """ raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a _set_autocommit() method') # ##### Generic transaction management methods ##### def get_autocommit(self): """ Check the autocommit state. """ self.ensure_connection() return self.autocommit def set_autocommit(self, autocommit, force_begin_transaction_with_broken_autocommit=False): """ Enable or disable autocommit. The usual way to start a transaction is to turn autocommit off. SQLite does not properly start a transaction when disabling autocommit. To avoid this buggy behavior and to actually enter a new transaction, an explcit BEGIN is required. Using force_begin_transaction_with_broken_autocommit=True will issue an explicit BEGIN with SQLite. This option will be ignored for other backends. """ self.validate_no_atomic_block() self.ensure_connection() start_transaction_under_autocommit = ( force_begin_transaction_with_broken_autocommit and not autocommit and self.features.autocommits_when_autocommit_is_off ) if start_transaction_under_autocommit: self._start_transaction_under_autocommit() else: self._set_autocommit(autocommit) self.autocommit = autocommit if autocommit and self.run_commit_hooks_on_set_autocommit_on: self.run_and_clear_commit_hooks() self.run_commit_hooks_on_set_autocommit_on = False def get_rollback(self): """ Get the "needs rollback" flag -- for *advanced use* only. """ if not self.in_atomic_block: raise TransactionManagementError( "The rollback flag doesn't work outside of an 'atomic' block.") return self.needs_rollback def set_rollback(self, rollback): """ Set or unset the "needs rollback" flag -- for *advanced use* only. """ if not self.in_atomic_block: raise TransactionManagementError( "The rollback flag doesn't work outside of an 'atomic' block.") self.needs_rollback = rollback def validate_no_atomic_block(self): """ Raise an error if an atomic block is active. """ if self.in_atomic_block: raise TransactionManagementError( "This is forbidden when an 'atomic' block is active.") def validate_no_broken_transaction(self): if self.needs_rollback: raise TransactionManagementError( "An error occurred in the current transaction. You can't " "execute queries until the end of the 'atomic' block.") # ##### Foreign key constraints checks handling ##### @contextmanager def constraint_checks_disabled(self): """ Context manager that disables foreign key constraint checking. """ disabled = self.disable_constraint_checking() try: yield finally: if disabled: self.enable_constraint_checking() def disable_constraint_checking(self): """ Backends can implement as needed to temporarily disable foreign key constraint checking. Should return True if the constraints were disabled and will need to be reenabled. """ return False def enable_constraint_checking(self): """ Backends can implement as needed to re-enable foreign key constraint checking. """ pass def check_constraints(self, table_names=None): """ Backends can override this method if they can apply constraint checking (e.g. via "SET CONSTRAINTS ALL IMMEDIATE"). Should raise an IntegrityError if any invalid foreign key references are encountered. """ pass # ##### Connection termination handling ##### def is_usable(self): """ Tests if the database connection is usable. This function may assume that self.connection is not None. Actual implementations should take care not to raise exceptions as that may prevent Django from recycling unusable connections. """ raise NotImplementedError( "subclasses of BaseDatabaseWrapper may require an is_usable() method") def close_if_unusable_or_obsolete(self): """ Closes the current connection if unrecoverable errors have occurred, or if it outlived its maximum age. """ if self.connection is not None: # If the application didn't restore the original autocommit setting, # don't take chances, drop the connection. if self.get_autocommit() != self.settings_dict['AUTOCOMMIT']: self.close() return # If an exception other than DataError or IntegrityError occurred # since the last commit / rollback, check if the connection works. if self.errors_occurred: if self.is_usable(): self.errors_occurred = False else: self.close() return if self.close_at is not None and time.time() >= self.close_at: self.close() return # ##### Thread safety handling ##### def validate_thread_sharing(self): """ Validates that the connection isn't accessed by another thread than the one which originally created it, unless the connection was explicitly authorized to be shared between threads (via the `allow_thread_sharing` property). Raises an exception if the validation fails. """ if not (self.allow_thread_sharing or self._thread_ident == thread.get_ident()): raise DatabaseError("DatabaseWrapper objects created in a " "thread can only be used in that same thread. The object " "with alias '%s' was created in thread id %s and this is " "thread id %s." % (self.alias, self._thread_ident, thread.get_ident())) # ##### Miscellaneous ##### def prepare_database(self): """ Hook to do any database check or preparation, generally called before migrating a project or an app. """ pass @cached_property def wrap_database_errors(self): """ Context manager and decorator that re-throws backend-specific database exceptions using Django's common wrappers. """ return DatabaseErrorWrapper(self) def make_debug_cursor(self, cursor): """ Creates a cursor that logs all queries in self.queries_log. """ return utils.CursorDebugWrapper(cursor, self) def make_cursor(self, cursor): """ Creates a cursor without debug logging. """ return utils.CursorWrapper(cursor, self) @contextmanager def temporary_connection(self): """ Context manager that ensures that a connection is established, and if it opened one, closes it to avoid leaving a dangling connection. This is useful for operations outside of the request-response cycle. Provides a cursor: with self.temporary_connection() as cursor: ... """ must_close = self.connection is None cursor = self.cursor() try: yield cursor finally: cursor.close() if must_close: self.close() @property def _nodb_connection(self): """ Return an alternative connection to be used when there is no need to access the main database, specifically for test db creation/deletion. This also prevents the production database from being exposed to potential child threads while (or after) the test database is destroyed. Refs #10868, #17786, #16969. """ settings_dict = self.settings_dict.copy() settings_dict['NAME'] = None nodb_connection = self.__class__( settings_dict, alias=NO_DB_ALIAS, allow_thread_sharing=False) return nodb_connection def _start_transaction_under_autocommit(self): """ Only required when autocommits_when_autocommit_is_off = True. """ raise NotImplementedError( 'subclasses of BaseDatabaseWrapper may require a ' '_start_transaction_under_autocommit() method' ) def schema_editor(self, *args, **kwargs): """ Returns a new instance of this backend's SchemaEditor. """ if self.SchemaEditorClass is None: raise NotImplementedError( 'The SchemaEditorClass attribute of this database wrapper is still None') return self.SchemaEditorClass(self, *args, **kwargs) def on_commit(self, func): if self.in_atomic_block: # Transaction in progress; save for execution on commit. self.run_on_commit.append((set(self.savepoint_ids), func)) elif not self.get_autocommit(): raise TransactionManagementError('on_commit() cannot be used in manual transaction management') else: # No transaction in progress and in autocommit mode; execute # immediately. func() def run_and_clear_commit_hooks(self): self.validate_no_atomic_block() try: while self.run_on_commit: sids, func = self.run_on_commit.pop(0) func() finally: self.run_on_commit = [] def copy(self, alias=None, allow_thread_sharing=None): """ Return a copy of this connection. For tests that require two connections to the same database. """ settings_dict = copy.deepcopy(self.settings_dict) if alias is None: alias = self.alias if allow_thread_sharing is None: allow_thread_sharing = self.allow_thread_sharing return type(self)(settings_dict, alias, allow_thread_sharing)