#!/usr/bin/env python3
# -*- coding: utf-8 -*-"
# Copyright 2021 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Search upstream commits in rebase database amd mark accordingly"""

from __future__ import print_function

from collections import defaultdict
# requires "pip install fuzzywuzzy"
import operator
import re
import subprocess
import time
import sqlite3

from common import chromeos_path
from common import rebasedb
from common import upstream_path
from common import upstreamdb
# from common import nextdb
from common import is_in_target

from fuzzywuzzy import fuzz    # pylint: disable=import-error
from fuzzywuzzy import process # pylint: disable=import-error

encoding = 'utf-8'

# List of all subjects, split into dictionary indexed by each word
# in the subject.
_alldescs = defaultdict(list)


def NOW():
    """Return current time"""
    return int(time.time())


def get_patch(path, psha):
    """Return patch associated with psha from repository, or None if not found"""

    patch = subprocess.check_output(
        ['git', '-C', path, 'show', "--format='%b'", '-U1', psha],
        encoding='utf-8', errors='ignore')
    i = re.search('^diff', patch, flags=re.MULTILINE).group()
    if i:
        ind = patch.index(i)
        return patch[ind:]
    return None


def patch_ratio(usha, lsha, ref=upstream_path):
    """compare patches

    Args:
        usha: First SHA to compare
        lsha: Second SHA to compare
        ref: optional repository path name

    Returns:
        Tuple with two different fuzzy matches

    Fuzzy match is applied to first 1,000 lines in each patch
    to avoid stalls. If one of the patches has more than 1,000
    lines, also compare the number of lines in each patch and
    return (0,0) if the mismatch is too significant.
    """

    lpatch = get_patch(ref, usha)
    if lpatch:
        upatch = get_patch(chromeos_path, lsha)
        if upatch:
            llen = lpatch.count('\n')
            ulen = upatch.count('\n')
            # Large patches: more than 20% difference in patch size is a mismatch
            if llen > 2000 or ulen > 2000:
                if abs(llen - ulen) > llen / 5:
                    return (0, 0)
            lpatch = '\n'.join(lpatch.splitlines()[:2000])
            upatch = '\n'.join(upatch.splitlines()[:2000])
            return (fuzz.ratio(upatch,
                               lpatch), fuzz.token_set_ratio(upatch, lpatch))
    return (0, 0)


def best_match(s):
    """Find best match for subject in _alldescs.

    Split provided subject into words. Search for subject in each of the
    word lists.

    Args:
        s: The subject to match.

    Returns:
        Best subject match as list (subject, score). If multiple subjects match
        with the same score, return first encountered match with this score.
    """

    matches = []
    s = re.sub(r'[^a-zA-Z0-9_/\s]+', ' ', s)
    for word in s.split():
        match = process.extractOne(
            s, _alldescs[word], scorer=fuzz.token_sort_ratio, score_cutoff=65)
        if match:
            matches.append(match)
    if not matches:
        return ('', 0)
    best = max(matches, key=operator.itemgetter(1))
    return (best[0], best[1])


def getallsubjects(db=upstreamdb):
    """Split subjects into a dictionary of of word-hashed lists.

    By searching the resulting lists, we can speed up processing
    significantly.

    Returns:
        _alldescs[] is populated.
    """

    global _alldescs # pylint: disable=global-statement

    _alldescs = defaultdict(list)
    db = sqlite3.connect(db)
    cu = db.cursor()
    cu.execute('select subject from commits')
    for subject, in cu.fetchall():
        wlist = re.sub(r'[^a-zA-Z0-9_\s]+', ' ', subject)
        words = wlist.split()
        for word in words:
            _alldescs[word].append(subject)
    db.close()


def update_commit(c, sha, disposition, reason, sscore=None, pscore=None):
    """Update a commit entry in the database if the disposition changes"""

    c.execute("select disposition from commits where sha='%s'" % sha)
    disp = c.fetchall()
    if not disp or disp != disposition:
        c.execute("UPDATE commits SET disposition=('%s') where sha='%s'" %
                  (disposition, sha))
        c.execute("UPDATE commits SET reason=('%s') where sha='%s'" %
                  (reason, sha))
        if sscore:
            c.execute("UPDATE commits SET sscore=%d where sha='%s'" %
                      (sscore, sha))
        if pscore:
            c.execute("UPDATE commits SET pscore=%d where sha='%s'" %
                      (pscore, sha))
        c.execute("UPDATE commits SET updated=('%d') where sha='%s'" %
                  (NOW(), sha))
    else:
        print('Registered disposition for sha %s: %s' % (sha, disp))
        print(
            "Not updating database for SHA '%s', requested disposition=%s, reason=%s"
            % (sha, disposition, reason))


def doit(db=upstreamdb, path=upstream_path, name='upstream'):
    """Do the actual work.

    Read all commits from database, compare against commits in provided
    database, and mark accordingly.
    """

    rp = re.compile(
        '(CHROMIUM: *|CHROMEOS: *|UPSTREAM: *|FROMGIT: *|FROMLIST: *|BACKPORT: *)+(.*)'
    )
    rpf = re.compile('(FIXUP: *|Fixup: *)(.*)')

    merge = sqlite3.connect(rebasedb)
    c = merge.cursor()
    c2 = merge.cursor()
    db = sqlite3.connect(db)
    cu = db.cursor()

    c.execute('select sha, patchid, subject, disposition from commits')
    for (sha, patchid, desc, disposition) in c.fetchall(): # pylint: disable=too-many-nested-blocks
        if disposition == 'drop':
            continue
        # First look for matching upstream patch ID
        cu.execute("SELECT sha, subject from commits where patchid is '%s'" %
                   patchid)
        fsha = cu.fetchone()
        if fsha:
            rsha = fsha[0]
            subject = fsha[1]
            c2.execute("UPDATE commits SET dsha=('%s') where sha='%s'" %
                       (rsha, sha))
            in_target = is_in_target(rsha)
            if in_target:
                disposition = 'drop'
            else:
                disposition = 'replace'
            print("Patch ID match for %s ('%s')" %
                  (sha, desc.replace("'", "''")))
            print("    Matching %s commit %s ('%s'), %s" %
                  (name, rsha, subject.replace("'", "''"), disposition))
            update_commit(c2, sha, disposition, 'upstream')
            continue

        m = rp.search(desc)
        mf = rpf.search(desc)
        if m:
            # print("Regex match for '%s'" % desc.replace("'", "''"))
            ndesc = m.group(2).replace("'", "''")
            rdesc = m.group(2)
            # print("    Match subject '%s'" % ndesc)
            cu.execute('select sha, subject, integrated from commits '
                       "where subject='%s'" % ndesc)
            fsha = cu.fetchone()
            if fsha:
                c2.execute("UPDATE commits SET dsha=('%s') where sha='%s'" %
                           (fsha[0], sha))
                in_target = is_in_target(fsha[2])
                if mf:
                    print("Regex match for %s '%s'" %
                          (sha, desc.replace("'", "''")))
                    print("    Match subject '%s'" % ndesc)
                    print('    FIXUP patch')
                    print("    Found matching %s commit %s ('%s'), drop" %
                          (name, fsha[0], fsha[1].replace("'", "''")))
                    update_commit(c2, sha, 'drop', '%s/fixup' % name, 100)
                    continue
                # print("    Upstream subject for %s matches %s" % (fsha[1], sha))
                # print("    Local subject: %s" % desc)
                # print("    Upstream subject: %s" % ndesc)
                # print("    In v4.9: %d" % fsha[2])
                if in_target:
                    disposition = 'drop'
                else:
                    disposition = 'replace'

                # This is a perfect match. Set sscore to 100.
                sscore = 100

                (ratio, setratio) = patch_ratio(fsha[0], sha, ref=path)
                pscore = (ratio + setratio) / 2

                # Like many others, 160 is a magic number derived from experiments.
                if ratio + setratio > 160:
                    reason = '%s/match' % name
                else:
                    reason = 'revisit'

                update_commit(c2, sha, disposition, reason, sscore, pscore)
            else:
                print("Regex match for '%s'" % desc.replace("'", "''"))
                print("    Match subject '%s'" % ndesc)
                print(
                    "    No match in %s for '%s' [marked %s], trying fuzzy match"
                    % (name, sha, disposition))
                (mdesc, result) = best_match(rdesc)
                if result == 0:
                    print('    No close match')
                    continue
                if result <= 75:
                    print('    Best candidate: %s' % mdesc)
                    print('    Basic subject match %d insufficient' % result)
                    # If the patch is tagged UPSTREAM:, but upstream does not have
                    # a matching subject, something is odd. Need to revisit.
                    if desc.startswith('UPSTREAM:'):
                        c2.execute(
                            "UPDATE commits SET reason=('revisit') where sha='%s'"
                            % sha)
                        c2.execute(
                            "UPDATE commits SET sscore=%d where sha='%s'" %
                            (result, sha))
                    continue
                # Use default ratio (not fuzz.token_sort_ratio) for further matching.
                result = fuzz.ratio(rdesc, mdesc)
                smatch = fuzz.token_set_ratio(rdesc, mdesc)
                print('    subject match results %d/%d' % (result, smatch))
                c2.execute("UPDATE commits SET sscore=%d where sha='%s'" %
                           ((result + smatch) / 2, sha))
                cu.execute('select sha, subject, integrated from commits '
                           "where subject='%s'" % mdesc.replace("'", "''"))
                fsha = cu.fetchone()
                if fsha:
                    c2.execute("UPDATE commits SET dsha=('%s') where sha='%s'" %
                               (fsha[0], sha))
                    in_target = is_in_target(fsha[2])
                    print("    Upstream candidate %s ('%s')" %
                          (fsha[0], fsha[1].replace("'", "''")))
                    if mf:
                        # We have:
                        #        sha is this patch
                        #        fsha[0] is the replacement candidate
                        c2.execute(
                            "select sha from commits where dsha is '%s'" %
                            fsha[0])
                        dsha = c2.fetchone()
                        if dsha:
                            print('    FIXUP: Found patch in %s as replacement. dropping')
                            update_commit(c2, sha, 'drop', 'revisit/fixup', 100)
                        else:
                            print('    FIXUP: No replacement target. Revisit.')
                            c2.execute(
                                "UPDATE commits SET reason=('revisit') where sha='%s'"
                                % sha)
                        continue
                    (ratio, setratio) = patch_ratio(fsha[0], sha)
                    c2.execute("UPDATE commits SET pscore=%d where sha='%s'" %
                               ((ratio + setratio) / 2, sha))
                    if ((result <= 90 or smatch < 98) and smatch != 100 and
                        (result <= 95 or smatch <= 95)):
                        # Compare subject strings after ':'.
                        # If there is a perfect match, look into patch contents after all
                        rdesc2 = re.sub(r'[\S]+:\s*', '', rdesc)
                        mdesc2 = re.sub(r'[\S]+:\s*', '', mdesc)
                        if rdesc2 != mdesc2:
                            print('    Subject match %d/%d insufficient' %
                                  (result, smatch))
                            c2.execute(
                                "UPDATE commits SET reason=('revisit') where sha='%s'"
                                % sha)
                            continue
                    c2.execute("select filename from files where sha is '%s'" %
                               sha)
                    lfilenames = c2.fetchall()
                    cu.execute("select filename from files where sha is '%s'" %
                               fsha[0])
                    ufilenames = cu.fetchall()
                    scrutiny = 0
                    if lfilenames != ufilenames:
                        print('    File name mismatch, increasing scrutiny')
                        scrutiny = 20
                    print('    patch match results %d/%d' % (ratio, setratio))
                    if (smatch < 100 and
                        (ratio <= 90 or
                         setratio <= 90)) or ratio <= 70 + scrutiny:
                        print('    code match %d/%d insufficient' %
                              (ratio, setratio))
                        print("    Mark sha '%s' for revisit" % sha)
                        c2.execute(
                            "UPDATE commits SET reason=('revisit') where sha='%s'"
                            % sha)
                        continue
                    # We have a match.
                    if in_target:
                        print("    Drop sha '%s' (close match)" % sha)
                        disposition = 'drop'
                        reason = '%s/match' % name
                    else:
                        print("    Replace sha '%s' with '%s' (close match)" %
                              (sha, fsha[0]))
                        disposition = 'replace'
                        reason = 'revisit'
                    update_commit(c2, sha, disposition, reason)
                else:
                    print("    NOTICE: missing match in %s for '%s'" %
                          (name, mdesc.replace("'", "''")))

    merge.commit()
    merge.close()
    db.close()


# First run against upstream (mainline).
getallsubjects()
doit()

# repeat against -next. This will generate a list of patches
# to be replaced with patches found in -next (which are probably
# a better match to future upstream patches). At the very least,
# this gives us an idea how many of the local patches are actually
# queued to the next kernel release.
# TODO: Check upstream/mainline and -next for Fixup: patches
# of patches which are going to be applied, and apply those
# as well.
# FIXME: This never really worked - next_path is not imported.
# We'll also have to update nextdb to match the format of
# upstreamdb if we really want this to work.
# if nextdb:
#     getallsubjects(nextdb)
#     doit(nextdb, next_path, 'next')