www.gitorious.org Git - mining-tools:mlstats.git/blob

1 #-*- coding:utf-8 -*-

3 #

4 # This program is free software; you can redistribute it and/or modify

5 # it under the terms of the GNU General Public License as published by

6 # the Free Software Foundation; either version 2 of the License, or

7 # (at your option) any later version.

8 #

9 # This program is distributed in the hope that it will be useful,

10 # but WITHOUT ANY WARRANTY; without even the implied warranty of

11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12 # GNU General Public License for more details.

13 #

14 # You should have received a copy of the GNU General Public License

15 # along with this program; if not, write to the Free Software

16 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

17 #

18 # Authors :

19 # Israel Herraiz <herraiz@gsyc.escet.urjc.es>

20 # Germán Poo-Caamaño <gpoo@gnome.org>

21

22 """

23 This module contains a basic SQL wrapper. It uses the standard

24 database API of Python, so any module may be used (just substitute

25 import MySQLdb for any other, for instance import PyGreSQL).

26

27 @authors: Israel Herraiz

28 @organization: Libresoft Research Group, Universidad Rey Juan Carlos

29 @copyright: Universidad Rey Juan Carlos (Madrid, Spain)

30 @license: GNU GPL version 2 or any later version

31 @contact: libresoft-tools-devel@lists.morfeo-project.org

32 """

33

34 import sys

35 import pprint

36 import psycopg2 as dbapi

37 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

38

39 from pymlstats.database import GenericDatabase

40

41

42 class Database(GenericDatabase):

43 def __init__(self, dbname='', username='', password='', hostname=None,

44 admin_user=None, admin_password=None):

45 GenericDatabase.__init__(self)

46

47 self.name = dbname

48 self.user = username

49 self.password = password

50 self.admin_user = admin_user

51 self.admin_password = admin_password

52 self.host = hostname

53

54 def connect(self):

55 dbname = 'dbname=%s' % (self.name)

56 user = 'user=%s' % (self.user or '')

57 password = 'password=%s' % (self.password or '')

58 host = 'host=%s' % (self.host or '')

59

60 dsn = ' '.join([dbname, user, password, host])

61

62 try:

63 db = dbapi.connect(dsn)

64 db.set_client_encoding('UTF8')

65 db.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)

66 except dbapi.OperationalError, e:

67 raise e

68

69 GenericDatabase.connect(self, db)

70

71 def insert_people(self, name, email, mailing_list_url):

72 try:

73 top_level_domain = email.split(".")[-1]

74 except IndexError:

75 top_level_domain = ''

76 try:

77 username, domain_name = email.split('@')

78 except ValueError:

79 username, domain_name = ('', '')

80

81 query_people = '''INSERT INTO people

82 (email_address, name, username,

83 domain_name, top_level_domain)

84 VALUES (%s, %s, %s, %s, %s);'''

85 from_values = [email, name, username, domain_name, top_level_domain]

86 try:

87 self.write_cursor.execute(query_people, from_values)

88 except dbapi.IntegrityError:

89 pass

90 except dbapi.DataError:

91 pprint.pprint(from_values)

92 raise

93

94 query_mailing_lists_people = '''INSERT INTO mailing_lists_people

95 (email_address, mailing_list_url)

96 VALUES (%s, %s);'''

97 mailing_lists_people_values = [email, mailing_list_url]

98 try:

99 self.write_cursor.execute(query_mailing_lists_people,

100 mailing_lists_people_values)

101 except dbapi.IntegrityError:

102 # Duplicate entry email address-mailing list url

103 pass

104 except dbapi.DataError:

105 pprint.pprint(mailing_lists_people_values)

106 raise

107

108 def store_messages(self, message_list, mailing_list_url):

109 query = 'SET CONSTRAINTS ALL DEFERRED'

110 self.write_cursor.execute(query)

111

112 stored_messages = 0

113 query_message = '''INSERT INTO messages (

114 message_id, is_response_of,

115 arrival_date, first_date, first_date_tz,

116 mailing_list, mailing_list_url,

117 subject, message_body)

118 VALUES (%(message-id)s, %(in-reply-to)s,

119 %(received)s, %(date)s, %(date_tz)s,

120 %(list-id)s, %(mailing_list_url)s,

121 %(subject)s, %(body)s);'''

122 query_m_people = '''INSERT INTO messages_people

123 (email_address, type_of_recipient, message_id)

124 VALUES (%s, %s, %s);'''

125

126 for m in message_list:

127 values = m

128 values['mailing_list_url'] = mailing_list_url

129

130 # FIXME: If primary key check fails, ignore and continue

131 msgs_people_value = {}

132 for header in ('from', 'to', 'cc'):

133 addresses = self.filter(m[header])

134 if not addresses:

135 continue

136

137 for name, email in addresses:

138 self.insert_people(name, email, mailing_list_url)

139 key = '%s-%s' % (header, email)

140 value = (email, header.capitalize(), m['message-id'])

141 msgs_people_value.setdefault(key, value)

142

143 # Write the rest of the message

144 try:

145 self.write_cursor.execute(query_message, values)

146 except dbapi.IntegrityError:

147 # Duplicated message

148 stored_messages -= 1

149 except dbapi.DataError:

150 pprint.pprint(values, sys.stderr)

151 raise

152 except:

153 error_message = """ERROR: Runtime error while trying to write

154 message with message-id '%s'. That message has not been written

155 to the database, but the execution has not been stopped. Please

156 report this failure including the message-id and the URL for

157 the mbox.""" % m['message-id']

158 stored_messages -= 1

159 # Write message to the stderr

160 print >> sys.stderr, error_message

161 print >> sys.stderr, query_message

162 pprint.pprint(values, sys.stderr)

163

164 for key, values in msgs_people_value.iteritems():

165 try:

166 self.write_cursor.execute(query_m_people, values)

167 except dbapi.IntegrityError:

168 # Duplicate entry email_address-to|cc-mailing list url

169 pass

170 except dbapi.DataError:

171 pprint.pprint(values, sys.stderr)

172 raise

173 self.dbobj.commit()

174 stored_messages += 1

175

176 # Check that everything is consistent

177 query = 'SET CONSTRAINTS ALL IMMEDIATE'

178 self.write_cursor.execute(query)

179 self.dbobj.commit()

180

181 return stored_messages