| # Copyright 1999-2014 Gentoo Foundation |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| from __future__ import division, unicode_literals |
| |
| import re |
| import sys |
| from portage.cache import fs_template |
| from portage.cache import cache_errors |
| from portage import os |
| from portage import _unicode_decode |
| from portage.util import writemsg |
| from portage.localization import _ |
| |
| if sys.hexversion >= 0x3000000: |
| # pylint: disable=W0622 |
| basestring = str |
| |
| class database(fs_template.FsBased): |
| |
| validation_chf = 'md5' |
| chf_types = ('md5', 'mtime') |
| |
| autocommits = False |
| synchronous = False |
| # cache_bytes is used together with page_size (set at sqlite build time) |
| # to calculate the number of pages requested, according to the following |
| # equation: cache_bytes = page_bytes * page_count |
| cache_bytes = 1024 * 1024 * 10 |
| |
| def __init__(self, *args, **config): |
| super(database, self).__init__(*args, **config) |
| self._import_sqlite() |
| self._allowed_keys = ["_eclasses_"] |
| self._allowed_keys.extend(self._known_keys) |
| self._allowed_keys.extend('_%s_' % k for k in self.chf_types) |
| self._allowed_keys_set = frozenset(self._allowed_keys) |
| self._allowed_keys = sorted(self._allowed_keys_set) |
| |
| self.location = os.path.join(self.location, |
| self.label.lstrip(os.path.sep).rstrip(os.path.sep)) |
| |
| if not self.readonly and not os.path.exists(self.location): |
| self._ensure_dirs() |
| |
| config.setdefault("autocommit", self.autocommits) |
| config.setdefault("cache_bytes", self.cache_bytes) |
| config.setdefault("synchronous", self.synchronous) |
| # Set longer timeout for throwing a "database is locked" exception. |
| # Default timeout in sqlite3 module is 5.0 seconds. |
| config.setdefault("timeout", 15) |
| self._db_init_connection(config) |
| self._db_init_structures() |
| |
| def _import_sqlite(self): |
| # sqlite3 is optional with >=python-2.5 |
| try: |
| import sqlite3 as db_module |
| except ImportError as e: |
| raise cache_errors.InitializationError(self.__class__, e) |
| |
| self._db_module = db_module |
| self._db_error = db_module.Error |
| |
| def _db_escape_string(self, s): |
| """meta escaping, returns quoted string for use in sql statements""" |
| if not isinstance(s, basestring): |
| # Avoid potential UnicodeEncodeError in python-2.x by |
| # only calling str() when it's absolutely necessary. |
| s = str(s) |
| return "'%s'" % s.replace("'", "''") |
| |
| def _db_init_connection(self, config): |
| self._dbpath = self.location + ".sqlite" |
| #if os.path.exists(self._dbpath): |
| # os.unlink(self._dbpath) |
| connection_kwargs = {} |
| connection_kwargs["timeout"] = config["timeout"] |
| try: |
| if not self.readonly: |
| self._ensure_dirs() |
| self._db_connection = self._db_module.connect( |
| database=_unicode_decode(self._dbpath), **connection_kwargs) |
| self._db_cursor = self._db_connection.cursor() |
| self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) |
| if not self.readonly and not self._ensure_access(self._dbpath): |
| raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath) |
| self._db_init_cache_size(config["cache_bytes"]) |
| self._db_init_synchronous(config["synchronous"]) |
| except self._db_error as e: |
| raise cache_errors.InitializationError(self.__class__, e) |
| |
| def _db_init_structures(self): |
| self._db_table = {} |
| self._db_table["packages"] = {} |
| mytable = "portage_packages" |
| self._db_table["packages"]["table_name"] = mytable |
| self._db_table["packages"]["package_id"] = "internal_db_package_id" |
| self._db_table["packages"]["package_key"] = "portage_package_key" |
| create_statement = [] |
| create_statement.append("CREATE TABLE") |
| create_statement.append(mytable) |
| create_statement.append("(") |
| table_parameters = [] |
| table_parameters.append("%s INTEGER PRIMARY KEY AUTOINCREMENT" % self._db_table["packages"]["package_id"]) |
| table_parameters.append("%s TEXT" % self._db_table["packages"]["package_key"]) |
| for k in self._allowed_keys: |
| table_parameters.append("%s TEXT" % k) |
| table_parameters.append("UNIQUE(%s)" % self._db_table["packages"]["package_key"]) |
| create_statement.append(",".join(table_parameters)) |
| create_statement.append(")") |
| |
| self._db_table["packages"]["create"] = " ".join(create_statement) |
| |
| cursor = self._db_cursor |
| for k, v in self._db_table.items(): |
| if self._db_table_exists(v["table_name"]): |
| create_statement = self._db_table_get_create(v["table_name"]) |
| table_ok, missing_keys = self._db_validate_create_statement(create_statement) |
| if table_ok: |
| if missing_keys: |
| for k in sorted(missing_keys): |
| cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % |
| (self._db_table["packages"]["table_name"], k)) |
| else: |
| writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) |
| cursor.execute("DROP TABLE %s" % v["table_name"]) |
| cursor.execute(v["create"]) |
| else: |
| cursor.execute(v["create"]) |
| |
| def _db_table_exists(self, table_name): |
| """return true/false dependant on a tbl existing""" |
| cursor = self._db_cursor |
| cursor.execute("SELECT name FROM sqlite_master WHERE type=\"table\" AND name=%s" % \ |
| self._db_escape_string(table_name)) |
| return len(cursor.fetchall()) == 1 |
| |
| def _db_table_get_create(self, table_name): |
| """return true/false dependant on a tbl existing""" |
| cursor = self._db_cursor |
| cursor.execute("SELECT sql FROM sqlite_master WHERE name=%s" % \ |
| self._db_escape_string(table_name)) |
| return cursor.fetchall()[0][0] |
| |
| def _db_validate_create_statement(self, statement): |
| missing_keys = None |
| if statement == self._db_table["packages"]["create"]: |
| return True, missing_keys |
| |
| m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % |
| (self._db_table["packages"]["table_name"], |
| self._db_table["packages"]["package_id"]), |
| statement) |
| if m is None: |
| return False, missing_keys |
| |
| unique_constraints = set([self._db_table["packages"]["package_key"]]) |
| missing_keys = set(self._allowed_keys) |
| unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') |
| column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') |
| for x in m.group(1).split(","): |
| m = column_re.match(x) |
| if m is not None: |
| missing_keys.discard(m.group(1)) |
| continue |
| m = unique_re.match(x) |
| if m is not None: |
| unique_constraints.discard(m.group(1)) |
| continue |
| |
| if unique_constraints: |
| return False, missing_keys |
| |
| return True, missing_keys |
| |
| def _db_init_cache_size(self, cache_bytes): |
| cursor = self._db_cursor |
| cursor.execute("PRAGMA page_size") |
| page_size=int(cursor.fetchone()[0]) |
| # number of pages, sqlite default is 2000 |
| cache_size = cache_bytes // page_size |
| cursor.execute("PRAGMA cache_size = %d" % cache_size) |
| cursor.execute("PRAGMA cache_size") |
| actual_cache_size = int(cursor.fetchone()[0]) |
| del cursor |
| if actual_cache_size != cache_size: |
| raise cache_errors.InitializationError(self.__class__,"actual cache_size = "+actual_cache_size+" does does not match requested size of "+cache_size) |
| |
| def _db_init_synchronous(self, synchronous): |
| cursor = self._db_cursor |
| cursor.execute("PRAGMA synchronous = %d" % synchronous) |
| cursor.execute("PRAGMA synchronous") |
| actual_synchronous=int(cursor.fetchone()[0]) |
| del cursor |
| if actual_synchronous!=synchronous: |
| raise cache_errors.InitializationError(self.__class__,"actual synchronous = "+actual_synchronous+" does does not match requested value of "+synchronous) |
| |
| def _getitem(self, cpv): |
| cursor = self._db_cursor |
| cursor.execute("select * from %s where %s=%s" % \ |
| (self._db_table["packages"]["table_name"], |
| self._db_table["packages"]["package_key"], |
| self._db_escape_string(cpv))) |
| result = cursor.fetchall() |
| if len(result) == 1: |
| pass |
| elif len(result) == 0: |
| raise KeyError(cpv) |
| else: |
| raise cache_errors.CacheCorruption(cpv, "key is not unique") |
| result = result[0] |
| d = {} |
| allowed_keys_set = self._allowed_keys_set |
| for column_index, column_info in enumerate(cursor.description): |
| k = column_info[0] |
| if k in allowed_keys_set: |
| v = result[column_index] |
| if v is None: |
| # This happens after a new empty column has been added. |
| v = "" |
| d[k] = v |
| |
| return d |
| |
| def _setitem(self, cpv, values): |
| update_statement = [] |
| update_statement.append("REPLACE INTO %s" % self._db_table["packages"]["table_name"]) |
| update_statement.append("(") |
| update_statement.append(','.join([self._db_table["packages"]["package_key"]] + self._allowed_keys)) |
| update_statement.append(")") |
| update_statement.append("VALUES") |
| update_statement.append("(") |
| values_parameters = [] |
| values_parameters.append(self._db_escape_string(cpv)) |
| for k in self._allowed_keys: |
| values_parameters.append(self._db_escape_string(values.get(k, ''))) |
| update_statement.append(",".join(values_parameters)) |
| update_statement.append(")") |
| cursor = self._db_cursor |
| try: |
| s = " ".join(update_statement) |
| cursor.execute(s) |
| except self._db_error as e: |
| writemsg("%s: %s\n" % (cpv, str(e))) |
| raise |
| |
| def commit(self): |
| self._db_connection.commit() |
| |
| def _delitem(self, cpv): |
| cursor = self._db_cursor |
| cursor.execute("DELETE FROM %s WHERE %s=%s" % \ |
| (self._db_table["packages"]["table_name"], |
| self._db_table["packages"]["package_key"], |
| self._db_escape_string(cpv))) |
| |
| def __contains__(self, cpv): |
| cursor = self._db_cursor |
| cursor.execute(" ".join( |
| ["SELECT %s FROM %s" % |
| (self._db_table["packages"]["package_id"], |
| self._db_table["packages"]["table_name"]), |
| "WHERE %s=%s" % ( |
| self._db_table["packages"]["package_key"], |
| self._db_escape_string(cpv))])) |
| result = cursor.fetchall() |
| if len(result) == 0: |
| return False |
| elif len(result) == 1: |
| return True |
| else: |
| raise cache_errors.CacheCorruption(cpv, "key is not unique") |
| |
| def __iter__(self): |
| """generator for walking the dir struct""" |
| cursor = self._db_cursor |
| cursor.execute("SELECT %s FROM %s" % \ |
| (self._db_table["packages"]["package_key"], |
| self._db_table["packages"]["table_name"])) |
| result = cursor.fetchall() |
| key_list = [x[0] for x in result] |
| del result |
| while key_list: |
| yield key_list.pop() |