#!/usr/bin/python

#-*- coding:utf-8 -*-

#

# This code is part of the LWN git data miner.

#

# Copyright 2007-9 LWN.net

# Copyright 2007-9 Jonathan Corbet <corbet@lwn.net>

# Copyright 2009 Germán Póo-Caamaño <gpoo@gnome.org>

#

# This file may be distributed under the terms of the GNU General

# Public License, version 2.

import database, csvdump, ConfigFile, reports

import getopt, datetime

import os, re, sys, rfc822, string

import file_types

import logparser

from patterns import patterns

Today = datetime.date.today()

#

# Control options.

#

MapUnknown = 0

DevReports = 1

DateStats = 0

AuthorSOBs = 1

FileFilter = None

CSVFile = None

CSVPrefix = None

AkpmOverLt = 0

DumpDB = 0

CFName = 'gitdm.config'

#

# Options:

#

# -a		Andrew Morton's signoffs shadow Linus's

# -c cfile	Specify a configuration file

# -d		Output individual developer stats

# -D		Output date statistics

# -h hfile	HTML output to hfile

# -l count	Maximum length for output lists

# -o file	File for text output

# -p prefix Prefix for CSV output

# -r pattern	Restrict to files matching pattern

# -s		Ignore author SOB lines

# -u		Map unknown employers to '(Unknown)'

# -x file.csv   Export raw statistics as CSV

# -z		Dump out the hacker database at completion

def ParseOpts ():

    global MapUnknown, DevReports

    global DateStats, AuthorSOBs, FileFilter, AkpmOverLt, DumpDB

    global CFName, CSVFile, CSVPrefix

    opts, rest = getopt.getopt (sys.argv[1:], 'adc:Dh:l:o:p:r:sux:z')

    for opt in opts:

        if opt[0] == '-a':

            AkpmOverLt = 1

        elif opt[0] == '-c':

            CFName = opt[1]

        elif opt[0] == '-d':

            DevReports = 0

        elif opt[0] == '-D':

            DateStats = 1

        elif opt[0] == '-h':

            reports.SetHTMLOutput (open (opt[1], 'w'))

        elif opt[0] == '-l':

            reports.SetMaxList (int (opt[1]))

        elif opt[0] == '-o':

            reports.SetOutput (open (opt[1], 'w'))

        elif opt[0] == '-p':

            CSVPrefix = opt[1]

        elif opt[0] == '-r':

            print 'Filter on "%s"' % (opt[1])

            FileFilter = re.compile (opt[1])

        elif opt[0] == '-s':

            AuthorSOBs = 0

        elif opt[0] == '-u':

            MapUnknown = 1

        elif opt[0] == '-x':

            CSVFile = open (opt[1], 'w')

            print "open output file " + opt[1] + "\n"

        elif opt[0] == '-z':

            DumpDB = 1

def LookupStoreHacker (name, email):

    email = database.RemapEmail (email)

    h = database.LookupEmail (email)

    if h: # already there

        return h

    elist = database.LookupEmployer (email, MapUnknown)

    h = database.LookupName (name)

    if h: # new email

        h.addemail (email, elist)

        return h

    return database.StoreHacker(name, elist, email)

#

# Date tracking.

#

DateMap = { }

def AddDateLines(date, lines):

    if lines > 1000000:

        print 'Skip big patch (%d)' % lines

        return

    try:

        DateMap[date] += lines

    except KeyError:

        DateMap[date] = lines

def PrintDateStats():

    dates = DateMap.keys ()

    dates.sort ()

    total = 0

    datef = open ('datelc', 'w')

    for date in dates:

        total += DateMap[date]

        datef.write ('%d/%02d/%02d %6d %7d\n' % (date.year, date.month, date.day,

                                    DateMap[date], total))

#

# Let's slowly try to move some smarts into this class.

#

class patch:

    (ADDED, REMOVED) = range (2)

    def __init__ (self, commit):

        self.commit = commit

        self.merge = self.added = self.removed = 0

        self.author = LookupStoreHacker('Unknown hacker', 'unknown@hacker.net')

        self.email = 'unknown@hacker.net'

        self.sobs = [ ]

        self.reviews = [ ]

        self.testers = [ ]

        self.reports = [ ]

        self.filetypes = {}

    def addreviewer (self, reviewer):

        self.reviews.append (reviewer)

    def addtester (self, tester):

        self.testers.append (tester)

    def addreporter (self, reporter):

        self.reports.append (reporter)

    def addfiletype (self, filetype, added, removed):

        if self.filetypes.has_key (filetype):

            self.filetypes[filetype][self.ADDED] += added

            self.filetypes[filetype][self.REMOVED] += removed

        else:

            self.filetypes[filetype] = [added, removed]

def parse_numstat(line, file_filter):

    """

        Receive a line of text, determine if fits a numstat line and

        parse the added and removed lines as well as the file type.

    """

    m = patterns['numstat'].match (line)

    if m:

        filename = m.group (3)

        # If we have a file filter, check for file lines.

        if file_filter and not file_filter.search (filename):

            return None, None, None, None

        try:

            added = int (m.group (1))

            removed = int (m.group (2))

        except ValueError:

            # A binary file (image, etc.) is marked with '-'

            added = removed = 0

        m = patterns['rename'].match (filename)

        if m:

            filename = '%s%s%s' % (m.group (1), m.group (3), m.group (4))

        filetype = file_types.guess_file_type (os.path.basename(filename))

        return filename, filetype, added, removed

    else:

        return None, None, None, None

#

# The core hack for grabbing the information about a changeset.

#

def grabpatch(logpatch):

    global TotalAdded, TotalRemoved, TotalChanged

    m = patterns['commit'].match (logpatch[0])

    if not m:

        return None

    p = patch(m.group (1))

    for Line in logpatch[1:]:

        #

        # Maybe it's an author line?

        #

        m = patterns['author'].match (Line)

        if m:

            p.email = database.RemapEmail (m.group (2))

            p.author = LookupStoreHacker(m.group (1), p.email)

            continue

        #

        # Could be a signed-off-by:

        #

        m = patterns['signed-off-by'].search (Line)

        if m:

            email = database.RemapEmail (m.group (2))

            sobber = LookupStoreHacker(m.group (1), email)

            if sobber != p.author or AuthorSOBs:

                p.sobs.append ((email, LookupStoreHacker(m.group (1), m.group (2))))

            continue

        #

        # Various other tags of interest.

        #

        # Reviewed-by:

        m = patterns['reviewed-by'].search (Line)

        if m:

            email = database.RemapEmail (m.group (2))

            p.addreviewer (LookupStoreHacker(m.group (1), email))

            continue

        # Tested-by:

        m = patterns['tested-by'].search (Line)

        if m:

            email = database.RemapEmail (m.group (2))

            p.addtester (LookupStoreHacker (m.group (1), email))

            p.author.testcredit (patch)

            continue

        # Reported-by:

        m = patterns['reported-by'].search (Line)

        if m:

            email = database.RemapEmail (m.group (2))

            p.addreporter (LookupStoreHacker (m.group (1), email))

            p.author.reportcredit (patch)

            continue

        # Reported-and-tested-by:

        m = patterns['reported-and-tested-by'].search (Line)

        if m:

            email = database.RemapEmail (m.group (2))

            h = LookupStoreHacker (m.group (1), email)

            p.addreporter (h)

            p.addtester (h)

            p.author.reportcredit (patch)

            p.author.testcredit (patch)

            continue

        #

        # If this one is a merge, make note of the fact.

        #

        m = patterns['merge'].match (Line)

        if m:

            p.merge = 1

            continue

        #

        # See if it's the date.

        #

        m = patterns['date'].match (Line)

        if m:

            dt = rfc822.parsedate(m.group (2))

            p.date = datetime.date (dt[0], dt[1], dt[2])

            if p.date > Today:

                sys.stderr.write ('Funky date: %s\n' % p.date)

                p.date = Today

            continue

        # Get the statistics (lines added/removes) using numstats

        # and without requiring a diff (--numstat instead -p)

        (filename, filetype, added, removed) = parse_numstat (Line, FileFilter)

        if filename:

            p.added += added

            p.removed += removed

            p.addfiletype (filetype, added, removed)

    if '@' in p.author.name:

        print '%s is an author name, probably not what you want' % p.author.name

    #

    # Record some global information - but only if this patch had

    # stuff which wasn't ignored.  This work should be done

    # elsewhere,

    #

    if ((p.added + p.removed) > 0 or not FileFilter) and not p.merge:

        TotalAdded += p.added

        TotalRemoved += p.removed

        TotalChanged += max (p.added, p.removed)

        AddDateLines (p.date, max (p.added, p.removed))

        empl = p.author.emailemployer (p.email, p.date)

        empl.AddCSet (p)

        if AkpmOverLt:

            TrimLTSOBs (p)

        for sobemail, sobber in p.sobs:

            empl = sobber.emailemployer (sobemail, p.date)

            empl.AddSOB()

    return p

#

# If this patch is signed off by both Andrew Morton and Linus Torvalds,

# remove the (redundant) Linus signoff.

#

def TrimLTSOBs (p):

    if Linus in p.sobs and Akpm in p.sobs:

        p.sobs.remove (Linus)

#

# Here starts the real program.

#

ParseOpts ()

#

# Read the config files.

#

ConfigFile.ConfigFile (CFName)

#

# Let's pre-seed the database with a couple of hackers

# we want to remember.

#

Linus = ('torvalds@linux-foundation.org',

         LookupStoreHacker ('Linus Torvalds', 'torvalds@linux-foundation.org'))

Akpm = ('akpm@linux-foundation.org',

        LookupStoreHacker ('Andrew Morton', 'akpm@linux-foundation.org'))

TotalChanged = TotalAdded = TotalRemoved = 0

#

# Snarf changesets.

#

print >> sys.stderr, 'Grabbing changesets...\r',

patches = logparser.LogPatchSplitter(sys.stdin)

printcount = CSCount = 0

for logpatch in patches:

    if (printcount % 50) == 0:

        print >> sys.stderr, 'Grabbing changesets...%d\r' % printcount,

    printcount += 1

    p = grabpatch(logpatch)

    if not p:

        break

#    if p.added > 100000 or p.removed > 100000:

#        print 'Skipping massive add', p.commit

#        continue

    if FileFilter and p.added == 0 and p.removed == 0:

        continue

    if not p.merge:

        p.author.addpatch (p)

        for sobemail, sob in p.sobs:

            sob.addsob (p)

        for hacker in p.reviews:

            hacker.addreview (p)

        for hacker in p.testers:

            hacker.addtested (p)

        for hacker in p.reports:

            hacker.addreport (p)

        CSCount += 1

    csvdump.AccumulatePatch (p)

    csvdump.store_patch (p)

print >> sys.stderr, 'Grabbing changesets...done       '

if DumpDB:

    database.DumpDB ()

#

# Say something

#

hlist = database.AllHackers ()

elist = database.AllEmployers ()

ndev = nempl = 0

for h in hlist:

    if len (h.patches) > 0:

        ndev += 1

for e in elist:

    if e.count > 0:

        nempl += 1

reports.Write ('Processed %d csets from %d developers\n' % (CSCount,

                                                            ndev))

reports.Write ('%d employers found\n' % (nempl))

reports.Write ('A total of %d lines added, %d removed (delta %d)\n' %

               (TotalAdded, TotalRemoved, TotalAdded - TotalRemoved))

if TotalChanged == 0:

    TotalChanged = 1 # HACK to avoid div by zero

if DateStats:

    PrintDateStats ()

    sys.exit(0)

if CSVPrefix:

    csvdump.save_csv (CSVPrefix)

if CSVFile:

    csvdump.OutputCSV (CSVFile)

    CSVFile.close ()

if DevReports:

    reports.DevReports (hlist, TotalChanged, CSCount, TotalRemoved)

reports.EmplReports (elist, TotalChanged, CSCount)