update importplugin and add subticket support.
branches/tl3_0/plugins/svn/ticketimportplugin/setup.py (diff) branches/tl3_0/plugins/svn/ticketimportplugin/talm_importer/__init__.py (diff) branches/tl3_0/plugins/svn/ticketimportplugin/talm_importer/test.py (diff) branches/tl3_0/plugins/svn/ticketimportplugin/talm_importer/importer.py (diff) branches/tl3_0/plugins/svn/ticketimportplugin/talm_importer/processors.py (diff) branches/tl3_0/plugins/svn/ticketimportplugin/talm_importer/ticket.py (diff) branches/tl3_0/plugins/svn/ticketimportplugin/talm_importer/readers.py (diff)| @@ -1,27 +1,27 @@ | ||
| 1 | -#!/usr/bin/env python | |
| 2 | -# -*- coding: utf-8 -*- | |
| 3 | -# | |
| 4 | - | |
| 5 | -# | |
| 6 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 7 | -# Author: Francois Granade - fg at nexb dot com | |
| 8 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 9 | -# | |
| 10 | - | |
| 11 | -from setuptools import setup | |
| 12 | - | |
| 13 | -PACKAGE='talm_importer' | |
| 14 | - | |
| 15 | -setup( | |
| 16 | - name='TicketImport', | |
| 17 | - version='0.7c', | |
| 18 | - author='Francois Granade', | |
| 19 | - author_email='fg@nexb.com', | |
| 20 | - url='http://nexb.com', | |
| 21 | - license='BSD', | |
| 22 | - description='Import CSV and Excel files', | |
| 23 | - zip_safe=True, | |
| 24 | - packages=[PACKAGE], | |
| 25 | - package_data={PACKAGE: ['templates/*.html']}, | |
| 26 | - entry_points={'trac.plugins': 'TicketImport = %s' % (PACKAGE)} | |
| 27 | - ) | |
| 1 | +#!/usr/bin/env python | |
| 2 | +# -*- coding: utf-8 -*- | |
| 3 | +# | |
| 4 | + | |
| 5 | +# | |
| 6 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 7 | +# Author: Francois Granade - fg at nexb dot com | |
| 8 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 9 | +# | |
| 10 | + | |
| 11 | +from setuptools import setup | |
| 12 | + | |
| 13 | +PACKAGE='talm_importer' | |
| 14 | + | |
| 15 | +setup( | |
| 16 | + name='TicketImport', | |
| 17 | + version='0.8', | |
| 18 | + author='Francois Granade', | |
| 19 | + author_email='fg@nexb.com', | |
| 20 | + url='http://nexb.com', | |
| 21 | + license='BSD', | |
| 22 | + description='Import CSV and Excel files', | |
| 23 | + zip_safe=True, | |
| 24 | + packages=[PACKAGE], | |
| 25 | + package_data={PACKAGE: ['templates/*.html']}, | |
| 26 | + entry_points={'trac.plugins': 'TicketImport = %s' % (PACKAGE)} | |
| 27 | + ) |
| @@ -1,8 +1,8 @@ | ||
| 1 | -# | |
| 2 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | -# Author: Francois Granade - fg at nexb dot com | |
| 4 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | -# | |
| 6 | - | |
| 7 | - | |
| 8 | -from talm_importer.importer import * | |
| 1 | +# | |
| 2 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | +# Author: Francois Granade - fg at nexb dot com | |
| 4 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | +# | |
| 6 | + | |
| 7 | + | |
| 8 | +from talm_importer.importer import * |
| @@ -1,262 +1,295 @@ | ||
| 1 | -# | |
| 2 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | -# Author: Francois Granade - fg at nexb dot com | |
| 4 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | -# | |
| 6 | - | |
| 7 | -import sys | |
| 8 | -import os | |
| 9 | -import tempfile | |
| 10 | -import shutil | |
| 11 | -import unittest | |
| 12 | -import pprint | |
| 13 | -import filecmp | |
| 14 | - | |
| 15 | -from trac.web.api import Request | |
| 16 | -from trac.env import Environment | |
| 17 | -from trac.core import TracError | |
| 18 | -#from trac.web.clearsilver import HDFWrapper | |
| 19 | - | |
| 20 | -from talm_importer.importer import ImportModule | |
| 21 | - | |
| 22 | -def _exec(cursor, sql, args = None): cursor.execute(sql, args) | |
| 23 | - | |
| 24 | -def _printme(something): pass # print something | |
| 25 | - | |
| 26 | - | |
| 27 | -class ImporterTestCase(unittest.TestCase): | |
| 28 | - | |
| 29 | - def _test_preview(self, env, filename): | |
| 30 | - req = Request({'SERVER_PORT': 0, 'SERVER_NAME': 'any', 'wsgi.url_scheme': 'any', 'wsgi.input': 'any', 'REQUEST_METHOD': 'GET' }, lambda x, y: _printme) | |
| 31 | - try: | |
| 32 | - from trac.test import MockPerm | |
| 33 | - req.perm = MockPerm() | |
| 34 | - except ImportError: | |
| 35 | - pass | |
| 36 | - req.authname = 'testuser' | |
| 37 | - #req.hdf = HDFWrapper([]) # replace by this if you want to generate HTML: req.hdf = HDFWrapper(loadpaths=chrome.get_all_templates_dirs()) | |
| 38 | - template, data, content_type = ImportModule(env)._do_preview(filename, 1, req) | |
| 39 | - #sys.stdout = tempstdout | |
| 40 | - #req.display(template, content_type or 'text/html') | |
| 41 | - #open('/tmp/out.html', 'w').write(req.hdf.render(template, None)) | |
| 42 | - pp = pprint.PrettyPrinter(indent=4) | |
| 43 | - return (pp.pformat(data) or '') + "\n" | |
| 44 | - | |
| 45 | - def _test_import(self, env, filename, sheet = 1): | |
| 46 | - req = Request({'SERVER_PORT': 0, 'SERVER_NAME': 'any', 'wsgi.url_scheme': 'any', 'wsgi.input': 'any', 'REQUEST_METHOD': 'GET' }, lambda x, y: _printme) | |
| 47 | - try: | |
| 48 | - from trac.test import MockPerm | |
| 49 | - req.perm = MockPerm() | |
| 50 | - except ImportError: | |
| 51 | - pass | |
| 52 | - req.authname = 'testuser' | |
| 53 | - #req.hdf = HDFWrapper([]) # replace by this if you want to generate HTML: req.hdf = HDFWrapper(loadpaths=chrome.get_all_templates_dirs()) | |
| 54 | - db = env.get_db_cnx() | |
| 55 | - cursor = db.cursor() | |
| 56 | - _exec(cursor, "select * from enum") | |
| 57 | - enums_before = cursor.fetchall() | |
| 58 | - _exec(cursor, "select * from component") | |
| 59 | - components_before = cursor.fetchall() | |
| 60 | - #print enums_before | |
| 61 | - # when testing, always use the same time so that the results are comparable | |
| 62 | - #print "importing " + filename + " with tickettime " + str(ImporterTestCase.TICKET_TIME) | |
| 63 | - template, data, content_type = ImportModule(env)._do_import(filename, sheet, req, filename, ImporterTestCase.TICKET_TIME) | |
| 64 | - #sys.stdout = tempstdout | |
| 65 | - #req.display(template, content_type or 'text/html') | |
| 66 | - #open('/tmp/out.html', 'w').write(req.hdf.render(template, None)) | |
| 67 | - _exec(cursor, "select * from ticket") | |
| 68 | - tickets = cursor.fetchall() | |
| 69 | - _exec(cursor, "select * from ticket_custom") | |
| 70 | - tickets_custom = cursor.fetchall() | |
| 71 | - _exec(cursor, "select * from ticket_change") | |
| 72 | - tickets_change = cursor.fetchall() | |
| 73 | - _exec(cursor, "select * from enum") | |
| 74 | - enums = [f for f in set(cursor.fetchall()) - set(enums_before)] | |
| 75 | - _exec(cursor, "select * from component") | |
| 76 | - components = [f for f in set(cursor.fetchall()) - set(components_before)] | |
| 77 | - pp = pprint.PrettyPrinter(indent=4) | |
| 78 | - return pp.pformat([ tickets, tickets_custom, tickets_change, enums, components ]) | |
| 79 | - | |
| 80 | - def _do_test(self, env, filename, testfun): | |
| 81 | - from os.path import join, dirname | |
| 82 | - testdir = join(dirname(dirname(dirname(testfolder))), 'test') | |
| 83 | - outfilename = join(testdir, filename + '.' + testfun.__name__ + '.out') | |
| 84 | - ctlfilename = join(testdir, filename + '.' + testfun.__name__ + '.ctl') | |
| 85 | - open(outfilename, 'w').write(testfun(env, join(testdir, filename))) | |
| 86 | - return filecmp.cmp(outfilename, ctlfilename) | |
| 87 | - | |
| 88 | - def _do_test_diffs(self, env, filename, testfun): | |
| 89 | - self._do_test(env, filename, testfun) | |
| 90 | - from os.path import join, dirname | |
| 91 | - testdir = join(dirname(dirname(dirname(testfolder))), 'test') | |
| 92 | - import sys | |
| 93 | - from difflib import Differ | |
| 94 | - d = Differ() | |
| 95 | - def readall(ext): return open(join(testdir, filename + ext), 'rb').readlines() | |
| 96 | - result = d.compare(readall('.' + testfun.__name__ + '.ctl'), readall('.' + testfun.__name__ + '.out')) | |
| 97 | - lines = [ line for line in result if line[0] != ' '] | |
| 98 | - #sys.stdout.writelines(lines) | |
| 99 | - self.assertEquals(0, len(lines)) | |
| 100 | - | |
| 101 | - def _do_test_with_exception(self, env, filename, testfun): | |
| 102 | - try: | |
| 103 | - self._do_test(env, filename, testfun) | |
| 104 | - except TracError, e: | |
| 105 | - return str(e) | |
| 106 | - | |
| 107 | - def _setup(self, configuration = None): | |
| 108 | - configuration = configuration or '[ticket-custom]\nmycustomfield = text\nmycustomfield.label = My Custom Field\nmycustomfield.order = 1\n' | |
| 109 | - | |
| 110 | - configuration += '\n[ticket]\ndefault_type = task\n' | |
| 111 | - | |
| 112 | - | |
| 113 | - instancedir = os.path.join(tempfile.gettempdir(), 'test-importer._preview') | |
| 114 | - if os.path.exists(instancedir): | |
| 115 | - shutil.rmtree(instancedir, False) | |
| 116 | - env = Environment(instancedir, create=True) | |
| 117 | - open(os.path.join(os.path.join(instancedir, 'conf'), 'trac.ini'), 'a').write('\n' + configuration + '\n') | |
| 118 | - db = env.get_db_cnx() | |
| 119 | - _exec(db.cursor(), "INSERT INTO permission VALUES ('anonymous', 'REPORT_ADMIN') ") | |
| 120 | - _exec(db.cursor(), "INSERT INTO permission VALUES ('anonymous', 'IMPORT_EXECUTE') ") | |
| 121 | - db.commit() | |
| 122 | - ImporterTestCase.TICKET_TIME = 1190909220 | |
| 123 | - return Environment(instancedir) | |
| 124 | - | |
| 125 | - def test_import_1(self): | |
| 126 | - env = self._setup() | |
| 127 | - db = env.get_db_cnx() | |
| 128 | - cursor = db.cursor() | |
| 129 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1245, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'sum2', u'', u'']) | |
| 130 | - db.commit() | |
| 131 | - self._do_test_diffs(env, 'Backlog-for-import.csv', self._test_preview) | |
| 132 | - self._do_test_diffs(env, 'simple.csv', self._test_preview) | |
| 133 | - self._do_test_diffs(env, 'simple.csv', self._test_preview) | |
| 134 | - self.assert_(self._do_test(env, 'simple.csv', self._test_import)) | |
| 135 | - # Run again, to make sure that the lookups are done corrctly | |
| 136 | - ImporterTestCase.TICKET_TIME = 1190909221 | |
| 137 | - self.assert_(self._do_test(env, 'simple-copy.csv', self._test_import)) | |
| 138 | - # import after modification should throw exception | |
| 139 | - _exec(cursor, "update ticket set changetime = " + str(ImporterTestCase.TICKET_TIME + 10) + " where id = 1245") | |
| 140 | - db.commit() | |
| 141 | - try: | |
| 142 | - pass | |
| 143 | - # TODO: this should throw an exception (a ticket has been modified between preview and import) | |
| 144 | - #_do_test(env, 'simple-copy.csv', self._test_import) | |
| 145 | - except TracError, err_string: | |
| 146 | - print err_string | |
| 147 | - #TODO: change the test case to modify the second or third row, to make sure that db.rollback() works | |
| 148 | - | |
| 149 | - def test_import_2(self): | |
| 150 | - env = self._setup() | |
| 151 | - self._do_test_diffs(env, 'various-charsets.xls', self._test_preview) | |
| 152 | - self.assert_(self._do_test(env, 'various-charsets.xls', self._test_import)) | |
| 153 | - | |
| 154 | - def test_import_3(self): | |
| 155 | - env = self._setup() | |
| 156 | - try: | |
| 157 | - self._do_test_diffs(env, 'with-id.csv', self._test_preview) | |
| 158 | - self.assert_(False) | |
| 159 | - except TracError, e: | |
| 160 | - self.assertEquals(str(e), 'Ticket 1 found in file, but not present in Trac: cannot import.') | |
| 161 | - | |
| 162 | - def test_import_4(self): | |
| 163 | - env = self._setup() | |
| 164 | - db = env.get_db_cnx() | |
| 165 | - cursor = db.cursor() | |
| 166 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'summary before change', u'', u'']) | |
| 167 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [2, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'summarybefore change', u'', u'']) | |
| 168 | - _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'mypriority', '1']) | |
| 169 | - _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'yourpriority', '2']) | |
| 170 | - _exec(cursor, "insert into component values (%s, %s, %s)", ['mycomp', '', '']) | |
| 171 | - _exec(cursor, "insert into component values (%s, %s, %s)", ['yourcomp', '', '']) | |
| 172 | - db.commit() | |
| 173 | - self._do_test_diffs(env, 'with-id.csv', self._test_preview) | |
| 174 | - self.assert_(self._do_test(env, 'with-id.csv', self._test_import)) | |
| 175 | - | |
| 176 | - def test_import_5(self): | |
| 177 | - env = self._setup() | |
| 178 | - db = env.get_db_cnx() | |
| 179 | - cursor = db.cursor() | |
| 180 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'a summary that is duplicated', u'', u'']) | |
| 181 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [2, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'a summary that is duplicated', u'', u'']) | |
| 182 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [3, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'a summary that is duplicated', u'', u'']) | |
| 183 | - db.commit() | |
| 184 | - self.assertEquals(self._do_test_with_exception(env, 'test-detect-duplicate-summary-in-trac.csv', self._test_preview), 'Tickets #1, #2 and #3 have the same summary "a summary that is duplicated" in Trac. Ticket reconciliation by summary can not be done. Please modify the summaries to ensure that they are unique.') | |
| 185 | - | |
| 186 | - def test_import_6(self): | |
| 187 | - env = self._setup() | |
| 188 | - self.assertEquals(self._do_test_with_exception(env, 'test-detect-duplicate-summary-in-spreadsheet.csv', self._test_import), 'Summary "test & integration" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.') | |
| 189 | - | |
| 190 | - def test_import_7(self): | |
| 191 | - self._setup() | |
| 192 | - instancedir = os.path.join(tempfile.gettempdir(), 'test-importer.tickets') | |
| 193 | - if os.path.exists(instancedir): | |
| 194 | - shutil.rmtree(instancedir, False) | |
| 195 | - _dbfile = os.path.join(os.path.join(instancedir, 'db'), 'trac.db') | |
| 196 | - env = Environment(instancedir, create=True) | |
| 197 | - os.remove(_dbfile) | |
| 198 | - shutil.copyfile(os.path.join(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(testfolder))), 'test'), 'tickets.db'), _dbfile) | |
| 199 | - open(os.path.join(os.path.join(instancedir, 'conf'), 'trac.ini'), 'a').write('\n[ticket-custom]\ndomain = text\ndomain.label = Domain\nstage = text\nstage.label = Stage\nusers = text\nusers.label = Users\n') | |
| 200 | - env = Environment(instancedir) | |
| 201 | - self.assert_(self._do_test(env, 'ticket-13.xls', self._test_import)) | |
| 202 | - | |
| 203 | - def test_import_with_ticket_types(self): | |
| 204 | - env = self._setup() | |
| 205 | - self._do_test_diffs(env, 'simple-with-type.csv', self._test_preview) | |
| 206 | - self.assert_(self._do_test(env, 'simple-with-type.csv', self._test_import)) | |
| 207 | - | |
| 208 | - def test_import_with_reconciliation_by_owner(self): | |
| 209 | - ''' | |
| 210 | - This test covers the two option flags "reconciliate_by_owner_also" and "skip_lines_with_empty_owner". | |
| 211 | - ''' | |
| 212 | - env = self._setup('\n[ticket-custom]\neffort = text\neffort.label = My Effort\n\n[importer]\nreconciliate_by_owner_also = true\nskip_lines_with_empty_owner = true\n') | |
| 213 | - self.assert_(self._do_test(env, 'same-summary-different-owners-for-reconcilation-with-owner.xls', self._test_import)) | |
| 214 | - | |
| 215 | - def test_import_csv_bug(self): | |
| 216 | - ''' | |
| 217 | - This test covers the same as precedent, plus a problem I had with CSV: | |
| 218 | - "TracError: Unable to read this file, does not seem to be a valid Excel or CSV file:newline inside string" | |
| 219 | - The problem disapeared when I fixed the issue in test_import_with_reconciliation_by_owner | |
| 220 | - ''' | |
| 221 | - env = self._setup('\n[ticket-custom]\neffort = text\neffort.label = My Effort\n\n[importer]\nreconciliate_by_owner_also = true\nskip_lines_with_empty_owner = true\n') | |
| 222 | - self.assert_(self._do_test(env, 'same-summary-different-owners-for-reconcilation-with-owner.csv', self._test_import)) | |
| 223 | - | |
| 224 | - def test_import_not_first_worksheet(self): | |
| 225 | - ''' | |
| 226 | - This test covers importing an index worksheet, plus a prb with an empty milestone: | |
| 227 | - File "/Users/francois/workspace/importer/talm_importer/importer.py", line 416, in _process | |
| 228 | - processor.process_new_lookups(newvalues) | |
| 229 | - File "/Users/francois/workspace/importer/talm_importer/processors.py", line 128, in process_new_lookups | |
| 230 | - lookup.insert() | |
| 231 | - File "/sw/lib/python2.4/site-packages/Trac-0.11b1-py2.4.egg/trac/ticket/model.py", line 650, in insert | |
| 232 | - assert self.name, 'Cannot create milestone with no name' | |
| 233 | - ''' | |
| 234 | - env = self._setup('\n[ticket-custom]\neffort = text\neffort.label = My Effort\n\n[importer]\nreconciliate_by_owner_also = true\nskip_lines_with_empty_owner = true\n') | |
| 235 | - def _test_import_fourth_sheet(env, filename): return self._test_import(env, filename, 4) | |
| 236 | - self.assert_(self._do_test(env, 'Backlog.xls', _test_import_fourth_sheet)) | |
| 237 | - | |
| 238 | - def test_import_with_id_called_id(self): | |
| 239 | - env = self._setup() | |
| 240 | - db = env.get_db_cnx() | |
| 241 | - cursor = db.cursor() | |
| 242 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'summary before change', u'', u'']) | |
| 243 | - _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [2, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'summarybefore change', u'', u'']) | |
| 244 | - _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'mypriority', '1']) | |
| 245 | - _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'yourpriority', '2']) | |
| 246 | - _exec(cursor, "insert into component values (%s, %s, %s)", ['mycomp', '', '']) | |
| 247 | - _exec(cursor, "insert into component values (%s, %s, %s)", ['yourcomp', '', '']) | |
| 248 | - db.commit() | |
| 249 | - self._do_test_diffs(env, 'with-id-called-id.csv', self._test_preview) | |
| 250 | - self.assert_(self._do_test(env, 'with-id-called-id.csv', self._test_import)) | |
| 251 | - | |
| 252 | - def test_import_non_ascii_ticket_4458(self): | |
| 253 | - env = self._setup() | |
| 254 | - self._do_test_diffs(env, 'non_ascii_ticket_4458.csv', self._test_preview) | |
| 255 | - | |
| 256 | - | |
| 257 | -def suite(): | |
| 258 | - return unittest.makeSuite(ImporterTestCase, 'test') | |
| 259 | - #return unittest.TestSuite( [ ImporterTestCase('test_import_2') ]) | |
| 260 | -if __name__ == '__main__': | |
| 261 | - testfolder = __file__ | |
| 262 | - unittest.main(defaultTest='suite') | |
| 1 | +# | |
| 2 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | +# Author: Francois Granade - fg at nexb dot com | |
| 4 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | +# | |
| 6 | + | |
| 7 | +import sys | |
| 8 | +import os | |
| 9 | +import tempfile | |
| 10 | +import shutil | |
| 11 | +import unittest | |
| 12 | +import pprint | |
| 13 | +import filecmp | |
| 14 | + | |
| 15 | +from trac.web.api import Request | |
| 16 | +from trac.env import Environment | |
| 17 | +from trac.core import TracError | |
| 18 | +#from trac.web.clearsilver import HDFWrapper | |
| 19 | + | |
| 20 | +from talm_importer.importer import ImportModule | |
| 21 | + | |
| 22 | +import trac | |
| 23 | +if trac.__version__.startswith('0.12'): | |
| 24 | + CTL_EXT = '-0.12.ctl' | |
| 25 | + TICKETS_DB = 'tickets-0.12.db' | |
| 26 | +else: | |
| 27 | + CTL_EXT = '.ctl' | |
| 28 | + TICKETS_DB = 'tickets.db' | |
| 29 | + | |
| 30 | + | |
| 31 | +def _exec(cursor, sql, args = None): cursor.execute(sql, args) | |
| 32 | + | |
| 33 | +def _printme(something): pass # print something | |
| 34 | + | |
| 35 | + | |
| 36 | +class ImporterTestCase(unittest.TestCase): | |
| 37 | + | |
| 38 | + def _test_preview(self, env, filename): | |
| 39 | + req = Request({'SERVER_PORT': 0, 'SERVER_NAME': 'any', 'wsgi.url_scheme': 'any', 'wsgi.input': 'any', 'REQUEST_METHOD': 'GET' }, lambda x, y: _printme) | |
| 40 | + try: | |
| 41 | + from trac.test import MockPerm | |
| 42 | + req.perm = MockPerm() | |
| 43 | + except ImportError: | |
| 44 | + pass | |
| 45 | + req.authname = 'testuser' | |
| 46 | + #req.hdf = HDFWrapper([]) # replace by this if you want to generate HTML: req.hdf = HDFWrapper(loadpaths=chrome.get_all_templates_dirs()) | |
| 47 | + template, data, content_type = ImportModule(env)._do_preview(filename, 1, req) | |
| 48 | + #sys.stdout = tempstdout | |
| 49 | + #req.display(template, content_type or 'text/html') | |
| 50 | + #open('/tmp/out.html', 'w').write(req.hdf.render(template, None)) | |
| 51 | + pp = pprint.PrettyPrinter(indent=4) | |
| 52 | + return (pp.pformat(data) or '') + "\n" | |
| 53 | + | |
| 54 | + def _test_import(self, env, filename, sheet = 1): | |
| 55 | + req = Request({'SERVER_PORT': 0, 'SERVER_NAME': 'any', 'wsgi.url_scheme': 'any', 'wsgi.input': 'any', 'REQUEST_METHOD': 'GET' }, lambda x, y: _printme) | |
| 56 | + try: | |
| 57 | + from trac.test import MockPerm | |
| 58 | + req.perm = MockPerm() | |
| 59 | + except ImportError: | |
| 60 | + pass | |
| 61 | + req.authname = 'testuser' | |
| 62 | + #req.hdf = HDFWrapper([]) # replace by this if you want to generate HTML: req.hdf = HDFWrapper(loadpaths=chrome.get_all_templates_dirs()) | |
| 63 | + db = env.get_db_cnx() | |
| 64 | + cursor = db.cursor() | |
| 65 | + _exec(cursor, "select * from enum") | |
| 66 | + enums_before = cursor.fetchall() | |
| 67 | + _exec(cursor, "select * from component") | |
| 68 | + components_before = cursor.fetchall() | |
| 69 | + #print enums_before | |
| 70 | + # when testing, always use the same time so that the results are comparable | |
| 71 | + #print "importing " + filename + " with tickettime " + str(ImporterTestCase.TICKET_TIME) | |
| 72 | + template, data, content_type = ImportModule(env)._do_import(filename, sheet, req, filename, ImporterTestCase.TICKET_TIME) | |
| 73 | + #sys.stdout = tempstdout | |
| 74 | + #req.display(template, content_type or 'text/html') | |
| 75 | + #open('/tmp/out.html', 'w').write(req.hdf.render(template, None)) | |
| 76 | + _exec(cursor, "select * from ticket") | |
| 77 | + tickets = cursor.fetchall() | |
| 78 | + _exec(cursor, "select * from ticket_custom") | |
| 79 | + tickets_custom = cursor.fetchall() | |
| 80 | + _exec(cursor, "select * from ticket_change") | |
| 81 | + tickets_change = cursor.fetchall() | |
| 82 | + _exec(cursor, "select * from enum") | |
| 83 | + enums = [f for f in set(cursor.fetchall()) - set(enums_before)] | |
| 84 | + _exec(cursor, "select * from component") | |
| 85 | + components = [f for f in set(cursor.fetchall()) - set(components_before)] | |
| 86 | + pp = pprint.PrettyPrinter(indent=4) | |
| 87 | + return pp.pformat([ tickets, tickets_custom, tickets_change, enums, components ]) | |
| 88 | + | |
| 89 | + def _do_test(self, env, filename, testfun): | |
| 90 | + from os.path import join, dirname | |
| 91 | + testdir = join(dirname(dirname(dirname(testfolder))), 'test') | |
| 92 | + outfilename = join(testdir, filename + '.' + testfun.__name__ + '.out') | |
| 93 | + ctlfilename = join(testdir, filename + '.' + testfun.__name__ + CTL_EXT) | |
| 94 | + open(outfilename, 'w').write(testfun(env, join(testdir, filename))) | |
| 95 | + return filecmp.cmp(outfilename, ctlfilename) | |
| 96 | + | |
| 97 | + def _do_test_diffs(self, env, filename, testfun): | |
| 98 | + self._do_test(env, filename, testfun) | |
| 99 | + from os.path import join, dirname | |
| 100 | + testdir = join(dirname(dirname(dirname(testfolder))), 'test') | |
| 101 | + import sys | |
| 102 | + from difflib import Differ | |
| 103 | + d = Differ() | |
| 104 | + def readall(ext): return open(join(testdir, filename + ext), 'rb').readlines() | |
| 105 | + result = d.compare(readall('.' + testfun.__name__ + CTL_EXT), | |
| 106 | + readall('.' + testfun.__name__ + '.out')) | |
| 107 | + lines = [ line for line in result if line[0] != ' '] | |
| 108 | + #sys.stdout.writelines(lines) | |
| 109 | + self.assertEquals(0, len(lines)) | |
| 110 | + | |
| 111 | + def _do_test_with_exception(self, env, filename, testfun): | |
| 112 | + try: | |
| 113 | + self._do_test(env, filename, testfun) | |
| 114 | + except TracError, e: | |
| 115 | + return str(e) | |
| 116 | + | |
| 117 | + def _setup(self, configuration = None): | |
| 118 | + configuration = configuration or '[ticket-custom]\nmycustomfield = text\nmycustomfield.label = My Custom Field\nmycustomfield.order = 1\n' | |
| 119 | + | |
| 120 | + configuration += '\n[ticket]\ndefault_type = task\n' | |
| 121 | + | |
| 122 | + | |
| 123 | + instancedir = os.path.join(tempfile.gettempdir(), 'test-importer._preview') | |
| 124 | + if os.path.exists(instancedir): | |
| 125 | + shutil.rmtree(instancedir, False) | |
| 126 | + env = Environment(instancedir, create=True) | |
| 127 | + open(os.path.join(os.path.join(instancedir, 'conf'), 'trac.ini'), 'a').write('\n' + configuration + '\n') | |
| 128 | + db = env.get_db_cnx() | |
| 129 | + _exec(db.cursor(), "INSERT INTO permission VALUES ('anonymous', 'REPORT_ADMIN') ") | |
| 130 | + _exec(db.cursor(), "INSERT INTO permission VALUES ('anonymous', 'IMPORT_EXECUTE') ") | |
| 131 | + db.commit() | |
| 132 | + ImporterTestCase.TICKET_TIME = 1190909220 | |
| 133 | + return Environment(instancedir) | |
| 134 | + | |
| 135 | + def test_import_1(self): | |
| 136 | + env = self._setup() | |
| 137 | + db = env.get_db_cnx() | |
| 138 | + cursor = db.cursor() | |
| 139 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1245, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'sum2', u'', u'']) | |
| 140 | + db.commit() | |
| 141 | + self._do_test_diffs(env, 'Backlog-for-import.csv', self._test_preview) | |
| 142 | + self._do_test_diffs(env, 'simple.csv', self._test_preview) | |
| 143 | + self._do_test_diffs(env, 'simple.csv', self._test_preview) | |
| 144 | + self.assert_(self._do_test(env, 'simple.csv', self._test_import)) | |
| 145 | + # Run again, to make sure that the lookups are done correctly | |
| 146 | + ImporterTestCase.TICKET_TIME = 1190909221 | |
| 147 | + self.assert_(self._do_test(env, 'simple-copy.csv', self._test_import)) | |
| 148 | + # import after modification should throw exception | |
| 149 | + _exec(cursor, "update ticket set changetime = " + str(ImporterTestCase.TICKET_TIME + 10) + " where id = 1245") | |
| 150 | + db.commit() | |
| 151 | + try: | |
| 152 | + pass | |
| 153 | + # TODO: this should throw an exception (a ticket has been modified between preview and import) | |
| 154 | + #_do_test(env, 'simple-copy.csv', self._test_import) | |
| 155 | + except TracError, err_string: | |
| 156 | + print err_string | |
| 157 | + #TODO: change the test case to modify the second or third row, to make sure that db.rollback() works | |
| 158 | + | |
| 159 | + def test_import_with_comments(self): | |
| 160 | + env = self._setup() | |
| 161 | + db = env.get_db_cnx() | |
| 162 | + cursor = db.cursor() | |
| 163 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1245, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'sum2', u'', u'']) | |
| 164 | + db.commit() | |
| 165 | + self._do_test_diffs(env, 'simple.csv', self._test_import) | |
| 166 | + self._do_test_diffs(env, 'simple_with_comments.csv', self._test_preview) | |
| 167 | + ImporterTestCase.TICKET_TIME = ImporterTestCase.TICKET_TIME + 100 | |
| 168 | + self._do_test_diffs(env, 'simple_with_comments.csv', self._test_import) | |
| 169 | + | |
| 170 | + def test_import_with_comments_and_description(self): | |
| 171 | + env = self._setup() | |
| 172 | + db = env.get_db_cnx() | |
| 173 | + cursor = db.cursor() | |
| 174 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1245, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'sum2', u'', u'']) | |
| 175 | + db.commit() | |
| 176 | + self._do_test_diffs(env, 'simple.csv', self._test_import) | |
| 177 | + self._do_test_diffs(env, 'simple_with_comments_and_description.csv', self._test_preview) | |
| 178 | + ImporterTestCase.TICKET_TIME = ImporterTestCase.TICKET_TIME + 100 | |
| 179 | + self._do_test_diffs(env, 'simple_with_comments_and_description.csv', self._test_import) | |
| 180 | + | |
| 181 | + | |
| 182 | + def test_import_2(self): | |
| 183 | + env = self._setup() | |
| 184 | + self._do_test_diffs(env, 'various-charsets.xls', self._test_preview) | |
| 185 | + self.assert_(self._do_test(env, 'various-charsets.xls', self._test_import)) | |
| 186 | + | |
| 187 | + def test_import_3(self): | |
| 188 | + env = self._setup() | |
| 189 | + try: | |
| 190 | + self._do_test_diffs(env, 'with-id.csv', self._test_preview) | |
| 191 | + self.assert_(False) | |
| 192 | + except TracError, e: | |
| 193 | + self.assertEquals(str(e), 'Ticket 1 found in file, but not present in Trac: cannot import.') | |
| 194 | + | |
| 195 | + def test_import_4(self): | |
| 196 | + env = self._setup() | |
| 197 | + db = env.get_db_cnx() | |
| 198 | + cursor = db.cursor() | |
| 199 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'summary before change', u'', u'']) | |
| 200 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [2, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'summarybefore change', u'', u'']) | |
| 201 | + _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'mypriority', '1']) | |
| 202 | + _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'yourpriority', '2']) | |
| 203 | + _exec(cursor, "insert into component values (%s, %s, %s)", ['mycomp', '', '']) | |
| 204 | + _exec(cursor, "insert into component values (%s, %s, %s)", ['yourcomp', '', '']) | |
| 205 | + db.commit() | |
| 206 | + self._do_test_diffs(env, 'with-id.csv', self._test_preview) | |
| 207 | + self.assert_(self._do_test(env, 'with-id.csv', self._test_import)) | |
| 208 | + | |
| 209 | + def test_import_5(self): | |
| 210 | + env = self._setup() | |
| 211 | + db = env.get_db_cnx() | |
| 212 | + cursor = db.cursor() | |
| 213 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'a summary that is duplicated', u'', u'']) | |
| 214 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [2, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'a summary that is duplicated', u'', u'']) | |
| 215 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [3, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'a summary that is duplicated', u'', u'']) | |
| 216 | + db.commit() | |
| 217 | + self.assertEquals(self._do_test_with_exception(env, 'test-detect-duplicate-summary-in-trac.csv', self._test_preview), 'Tickets #1, #2 and #3 have the same summary "a summary that is duplicated" in Trac. Ticket reconciliation by summary can not be done. Please modify the summaries to ensure that they are unique.') | |
| 218 | + | |
| 219 | + def test_import_6(self): | |
| 220 | + env = self._setup() | |
| 221 | + self.assertEquals(self._do_test_with_exception(env, 'test-detect-duplicate-summary-in-spreadsheet.csv', self._test_import), 'Summary "test & integration" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.') | |
| 222 | + | |
| 223 | + def test_import_7(self): | |
| 224 | + self._setup() | |
| 225 | + instancedir = os.path.join(tempfile.gettempdir(), 'test-importer.tickets') | |
| 226 | + if os.path.exists(instancedir): | |
| 227 | + shutil.rmtree(instancedir, False) | |
| 228 | + _dbfile = os.path.join(os.path.join(instancedir, 'db'), 'trac.db') | |
| 229 | + env = Environment(instancedir, create=True) | |
| 230 | + os.remove(_dbfile) | |
| 231 | + shutil.copyfile(os.path.join(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(testfolder))), 'test'), TICKETS_DB), _dbfile) | |
| 232 | + open(os.path.join(os.path.join(instancedir, 'conf'), 'trac.ini'), 'a').write('\n[ticket-custom]\ndomain = text\ndomain.label = Domain\nstage = text\nstage.label = Stage\nusers = text\nusers.label = Users\n') | |
| 233 | + env = Environment(instancedir) | |
| 234 | + self.assert_(self._do_test(env, 'ticket-13.xls', self._test_import)) | |
| 235 | + | |
| 236 | + def test_import_with_ticket_types(self): | |
| 237 | + env = self._setup() | |
| 238 | + self._do_test_diffs(env, 'simple-with-type.csv', self._test_preview) | |
| 239 | + self.assert_(self._do_test(env, 'simple-with-type.csv', self._test_import)) | |
| 240 | + | |
| 241 | + def test_import_with_reconciliation_by_owner(self): | |
| 242 | + ''' | |
| 243 | + This test covers the two option flags "reconciliate_by_owner_also" and "skip_lines_with_empty_owner". | |
| 244 | + ''' | |
| 245 | + env = self._setup('\n[ticket-custom]\neffort = text\neffort.label = My Effort\n\n[importer]\nreconciliate_by_owner_also = true\nskip_lines_with_empty_owner = true\n') | |
| 246 | + self.assert_(self._do_test(env, 'same-summary-different-owners-for-reconcilation-with-owner.xls', self._test_import)) | |
| 247 | + | |
| 248 | + def test_import_csv_bug(self): | |
| 249 | + ''' | |
| 250 | + This test covers the same as precedent, plus a problem I had with CSV: | |
| 251 | + "TracError: Unable to read this file, does not seem to be a valid Excel or CSV file:newline inside string" | |
| 252 | + The problem disapeared when I fixed the issue in test_import_with_reconciliation_by_owner | |
| 253 | + ''' | |
| 254 | + env = self._setup('\n[ticket-custom]\neffort = text\neffort.label = My Effort\n\n[importer]\nreconciliate_by_owner_also = true\nskip_lines_with_empty_owner = true\n') | |
| 255 | + self.assert_(self._do_test(env, 'same-summary-different-owners-for-reconcilation-with-owner.csv', self._test_import)) | |
| 256 | + | |
| 257 | + def test_import_not_first_worksheet(self): | |
| 258 | + ''' | |
| 259 | + This test covers importing an index worksheet, plus a prb with an empty milestone: | |
| 260 | + File "/Users/francois/workspace/importer/talm_importer/importer.py", line 416, in _process | |
| 261 | + processor.process_new_lookups(newvalues) | |
| 262 | + File "/Users/francois/workspace/importer/talm_importer/processors.py", line 128, in process_new_lookups | |
| 263 | + lookup.insert() | |
| 264 | + File "/sw/lib/python2.4/site-packages/Trac-0.11b1-py2.4.egg/trac/ticket/model.py", line 650, in insert | |
| 265 | + assert self.name, 'Cannot create milestone with no name' | |
| 266 | + ''' | |
| 267 | + env = self._setup('\n[ticket-custom]\neffort = text\neffort.label = My Effort\n\n[importer]\nreconciliate_by_owner_also = true\nskip_lines_with_empty_owner = true\n') | |
| 268 | + def _test_import_fourth_sheet(env, filename): return self._test_import(env, filename, 4) | |
| 269 | + self.assert_(self._do_test(env, 'Backlog.xls', _test_import_fourth_sheet)) | |
| 270 | + | |
| 271 | + def test_import_with_id_called_id(self): | |
| 272 | + env = self._setup() | |
| 273 | + db = env.get_db_cnx() | |
| 274 | + cursor = db.cursor() | |
| 275 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [1, u'defect', 1191377630, 1191377630, u'component1', None, u'major', u'somebody', u'anonymous', u'', u'', u'', u'new', None, u'summary before change', u'', u'']) | |
| 276 | + _exec(cursor, "insert into ticket values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [2, u'defect', 1191377630, 1191377630, u'component2', None, u'major', u'somebody2', u'anonymous2', u'', u'', u'', u'new', None, u'summarybefore change', u'', u'']) | |
| 277 | + _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'mypriority', '1']) | |
| 278 | + _exec(cursor, "insert into enum values (%s, %s, %s)", ['priority', 'yourpriority', '2']) | |
| 279 | + _exec(cursor, "insert into component values (%s, %s, %s)", ['mycomp', '', '']) | |
| 280 | + _exec(cursor, "insert into component values (%s, %s, %s)", ['yourcomp', '', '']) | |
| 281 | + db.commit() | |
| 282 | + self._do_test_diffs(env, 'with-id-called-id.csv', self._test_preview) | |
| 283 | + self.assert_(self._do_test(env, 'with-id-called-id.csv', self._test_import)) | |
| 284 | + | |
| 285 | + def test_import_non_ascii_ticket_4458(self): | |
| 286 | + env = self._setup() | |
| 287 | + self._do_test_diffs(env, 'non_ascii_ticket_4458.csv', self._test_preview) | |
| 288 | + | |
| 289 | + | |
| 290 | +def suite(): | |
| 291 | + return unittest.makeSuite(ImporterTestCase, 'test') | |
| 292 | + #return unittest.TestSuite( [ ImporterTestCase('test_import_7') ]) | |
| 293 | +if __name__ == '__main__': | |
| 294 | + testfolder = __file__ | |
| 295 | + unittest.main(defaultTest='suite') |
| @@ -1,452 +1,468 @@ | ||
| 1 | - | |
| 2 | -# | |
| 3 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 4 | -# Author: Francois Granade - fg at nexb dot com | |
| 5 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 6 | -# | |
| 7 | - | |
| 8 | -import os | |
| 9 | -import re | |
| 10 | -import shutil | |
| 11 | -import tempfile | |
| 12 | -import time | |
| 13 | -import unicodedata | |
| 14 | - | |
| 15 | -from trac.core import * | |
| 16 | -from trac.attachment import AttachmentModule | |
| 17 | -from trac.core import Component | |
| 18 | -from trac.perm import IPermissionRequestor | |
| 19 | -from trac.ticket import TicketSystem | |
| 20 | -from trac.ticket import model | |
| 21 | -from trac.util import get_reporter_id | |
| 22 | -from trac.util.html import html | |
| 23 | -from trac.web import IRequestHandler | |
| 24 | -from trac.web.chrome import INavigationContributor, ITemplateProvider | |
| 25 | - | |
| 26 | -from talm_importer.processors import ImportProcessor | |
| 27 | -from talm_importer.processors import PreviewProcessor | |
| 28 | -from talm_importer.readers import get_reader | |
| 29 | - | |
| 30 | - | |
| 31 | -class ImportModule(Component): | |
| 32 | - | |
| 33 | - implements(INavigationContributor, IPermissionRequestor, IRequestHandler, ITemplateProvider) | |
| 34 | - | |
| 35 | - # INavigationContributor methods | |
| 36 | - def get_active_navigation_item(self, req): | |
| 37 | - return 'importer' | |
| 38 | - | |
| 39 | - def get_navigation_items(self, req): | |
| 40 | - if not req.perm.has_permission('IMPORT_EXECUTE'): | |
| 41 | - return | |
| 42 | - yield ('mainnav', 'importer', | |
| 43 | - html.a('インポート', href=req.href.importer())) | |
| 44 | - | |
| 45 | - # IPermissionRequestor methods | |
| 46 | - def get_permission_actions(self): | |
| 47 | - return ['IMPORT_EXECUTE'] | |
| 48 | - | |
| 49 | - # IRequestHandler methods | |
| 50 | - | |
| 51 | - def match_request(self, req): | |
| 52 | - match = re.match(r'/importer(?:/([0-9]+))?', req.path_info) | |
| 53 | - if match: | |
| 54 | - return True | |
| 55 | - | |
| 56 | - def process_request(self, req): | |
| 57 | - req.perm.assert_permission('IMPORT_EXECUTE') | |
| 58 | - action = req.args.get('action', 'other') | |
| 59 | - | |
| 60 | - if req.args.has_key('cancel'): | |
| 61 | - req.redirect(req.href('importer')) | |
| 62 | - | |
| 63 | - if action == 'upload' and req.method == 'POST': | |
| 64 | - req.session['uploadedfile'] = None | |
| 65 | - uploadedfilename, uploadedfile = self._save_uploaded_file(req) | |
| 66 | - req.session['sheet'] = req.args['sheet'] | |
| 67 | - req.session['uploadedfile'] = uploadedfile | |
| 68 | - req.session['uploadedfilename'] = uploadedfilename | |
| 69 | - req.session['tickettime'] = str(int(time.time())) | |
| 70 | - return self._do_preview(uploadedfile, int(req.session['sheet']), req) | |
| 71 | - elif action == 'import' and req.method == 'POST': | |
| 72 | - tickettime = int(req.session['tickettime']) | |
| 73 | - if tickettime == 0: | |
| 74 | - raise TracError('No time set since preview, unable to import: please upload the file again') | |
| 75 | - | |
| 76 | - return self._do_import(req.session['uploadedfile'], int(req.session['sheet']), req, req.session['uploadedfilename'], tickettime) | |
| 77 | - | |
| 78 | - else: | |
| 79 | - req.session['uploadedfile'] = None | |
| 80 | - req.session['uploadedfilename'] = None | |
| 81 | - | |
| 82 | - data = { 'reconciliate_by_owner_also': self._reconciliate_by_owner_also(), | |
| 83 | - 'fields': ['ticket or id'] + [field['name'] for field in TicketSystem(self.env).get_ticket_fields()] } | |
| 84 | - | |
| 85 | - return 'importer.html', data, None | |
| 86 | - | |
| 87 | - # ITemplateProvider | |
| 88 | - | |
| 89 | - def get_htdocs_dirs(self): | |
| 90 | - """Return the absolute path of a directory containing additional | |
| 91 | - static resources (such as images, style sheets, etc). | |
| 92 | - """ | |
| 93 | - return [] | |
| 94 | - | |
| 95 | - def get_templates_dirs(self): | |
| 96 | - """Return the absolute path of the directory containing the provided | |
| 97 | - ClearSilver templates. | |
| 98 | - """ | |
| 99 | - from pkg_resources import resource_filename | |
| 100 | - return [resource_filename(__name__, 'templates')] | |
| 101 | - | |
| 102 | - # Internal methods | |
| 103 | - | |
| 104 | - def _do_preview(self, uploadedfile, sheet, req): | |
| 105 | - filereader = get_reader(uploadedfile, sheet) | |
| 106 | - try: | |
| 107 | - return self._process(filereader, get_reporter_id(req), PreviewProcessor(self.env, req)) | |
| 108 | - finally: | |
| 109 | - filereader.close() | |
| 110 | - | |
| 111 | - def _do_import(self, uploadedfile, sheet, req, uploadedfilename, tickettime): | |
| 112 | - filereader = get_reader(uploadedfile, sheet) | |
| 113 | - try: | |
| 114 | - try: | |
| 115 | - return self._process(filereader, get_reporter_id(req), ImportProcessor(self.env, req, uploadedfilename, tickettime)) | |
| 116 | - finally: | |
| 117 | - filereader.close() | |
| 118 | - except: | |
| 119 | - # Unlock the database. This is not really tested, but seems reasonable. TODO: test or verify this | |
| 120 | - self.env.get_db_cnx().rollback() | |
| 121 | - raise | |
| 122 | - | |
| 123 | - | |
| 124 | - def _save_uploaded_file(self, req): | |
| 125 | - req.perm.assert_permission('IMPORT_EXECUTE') | |
| 126 | - | |
| 127 | - upload = req.args['import-file'] | |
| 128 | - if not hasattr(upload, 'filename') or not upload.filename: | |
| 129 | - raise TracError('No file uploaded') | |
| 130 | - if hasattr(upload.file, 'fileno'): | |
| 131 | - size = os.fstat(upload.file.fileno())[6] | |
| 132 | - else: | |
| 133 | - upload.file.seek(0, 2) # seek to end of file | |
| 134 | - size = upload.file.tell() | |
| 135 | - upload.file.seek(0) | |
| 136 | - if size == 0: | |
| 137 | - raise TracError("Can't upload empty file") | |
| 138 | - | |
| 139 | - # Maximum file size (in bytes) | |
| 140 | - max_size = AttachmentModule.max_size | |
| 141 | - if max_size >= 0 and size > max_size: | |
| 142 | - raise TracError('Maximum file size (same as attachment size, set in trac.ini configuration file): %d bytes' % max_size, | |
| 143 | - 'Upload failed') | |
| 144 | - | |
| 145 | - # We try to normalize the filename to unicode NFC if we can. | |
| 146 | - # Files uploaded from OS X might be in NFD. | |
| 147 | - filename = unicodedata.normalize('NFC', unicode(upload.filename, | |
| 148 | - 'utf-8')) | |
| 149 | - filename = filename.replace('\\', '/').replace(':', '/') | |
| 150 | - filename = os.path.basename(filename) | |
| 151 | - if not filename: | |
| 152 | - raise TracError('No file uploaded') | |
| 153 | - | |
| 154 | - return filename, self._savedata(upload.file) | |
| 155 | - | |
| 156 | - | |
| 157 | - def _savedata(self, fileobj): | |
| 158 | - | |
| 159 | - # temp folder | |
| 160 | - tempuploadedfile = tempfile.mktemp() | |
| 161 | - | |
| 162 | - flags = os.O_CREAT + os.O_WRONLY + os.O_EXCL | |
| 163 | - if hasattr(os, 'O_BINARY'): | |
| 164 | - flags += os.O_BINARY | |
| 165 | - targetfile = os.fdopen(os.open(tempuploadedfile, flags), 'w') | |
| 166 | - | |
| 167 | - try: | |
| 168 | - shutil.copyfileobj(fileobj, targetfile) | |
| 169 | - finally: | |
| 170 | - targetfile.close() | |
| 171 | - return tempuploadedfile | |
| 172 | - | |
| 173 | - | |
| 174 | - | |
| 175 | - def _process(self, filereader, reporter, processor): | |
| 176 | - tracfields = [field['name'] for field in TicketSystem(self.env).get_ticket_fields()] | |
| 177 | - tracfields = [ 'ticket', 'id' ] + tracfields | |
| 178 | - customfields = [field['name'] for field in TicketSystem(self.env).get_custom_fields()] | |
| 179 | - | |
| 180 | - columns, rows = filereader.readers() | |
| 181 | - | |
| 182 | - # defensive: columns could be non-string, make sure they are | |
| 183 | - columns = map(str, columns) | |
| 184 | - | |
| 185 | - importedfields = [f for f in columns if f.lower() in tracfields] | |
| 186 | - notimportedfields = [f for f in columns if f.lower() not in tracfields and f.lower() != 'ticket' and f.lower() != 'id'] | |
| 187 | - | |
| 188 | - lowercaseimportedfields = [f.lower() for f in importedfields] | |
| 189 | - | |
| 190 | - idcolumn = None | |
| 191 | - | |
| 192 | - if 'ticket' in lowercaseimportedfields and 'id' in lowercaseimportedfields: | |
| 193 | - raise TracError, 'The first line of the worksheet contains both \'ticket\', and an \'id\' field name. Only one of them is needed to perform the import. Please check the file and try again.' | |
| 194 | - | |
| 195 | - ownercolumn = None | |
| 196 | - if 'ticket' in lowercaseimportedfields: | |
| 197 | - idcolumn = self._find_case_insensitive('ticket', importedfields) | |
| 198 | - elif 'id' in lowercaseimportedfields: | |
| 199 | - idcolumn = self._find_case_insensitive('id', importedfields) | |
| 200 | - elif 'summary' in lowercaseimportedfields: | |
| 201 | - summarycolumn = self._find_case_insensitive('summary', importedfields) | |
| 202 | - ownercolumn = self._reconciliate_by_owner_also() and self._find_case_insensitive('owner', importedfields) or None | |
| 203 | - else: | |
| 204 | - raise TracError, 'The first line of the worksheet contains neither a \'ticket\', an \'id\' nor a \'summary\' field name. At least one of them is needed to perform the import. Please check the file and try again.' | |
| 205 | - | |
| 206 | - # start TODO: this is too complex, it should be replaced by a call to TicketSystem(env).get_ticket_fields() | |
| 207 | - | |
| 208 | - # The fields that we will have to set a value for, if: | |
| 209 | - # - they are not in the imported fields, and | |
| 210 | - # - they are not set in the default values of the Ticket class, and | |
| 211 | - # - they shouldn't be set to empty | |
| 212 | - # if 'set' is true, this will be the value that will be set by default (even if the default value in the Ticket class is different) | |
| 213 | - # if 'set' is false, the value is computed by Trac and we don't have anything to do | |
| 214 | - computedfields = {'status': { 'value':'new', 'set': True }, | |
| 215 | - 'resolution' : { 'value': "''(None)''", 'set': False }, | |
| 216 | - 'reporter' : { 'value': reporter, 'set': True }, | |
| 217 | - 'time' : { 'value': "''(now)''", 'set': False }, | |
| 218 | - 'changetime' : { 'value': "''(now)''", 'set': False } } | |
| 219 | - | |
| 220 | - if 'owner' not in lowercaseimportedfields and 'component' in lowercaseimportedfields: | |
| 221 | - computedfields['owner'] = {} | |
| 222 | - computedfields['owner']['value'] = 'Computed from component' | |
| 223 | - computedfields['owner']['set'] = False | |
| 224 | - | |
| 225 | - # to get the compulted default values | |
| 226 | - from ticket import PatchedTicket | |
| 227 | - ticket = PatchedTicket(self.env) | |
| 228 | - | |
| 229 | - for f in [ 'type', 'cc' , 'url', 'description', 'keywords', 'component' , 'severity' , 'priority' , 'version', 'milestone' ] + customfields: | |
| 230 | - if f in ticket.values: | |
| 231 | - computedfields[f] = {} | |
| 232 | - computedfields[f]['value'] = ticket.values[f] | |
| 233 | - computedfields[f]['set'] = False | |
| 234 | - else: | |
| 235 | - computedfields[f] = None | |
| 236 | - | |
| 237 | - processor.start(importedfields, ownercolumn != None) | |
| 238 | - | |
| 239 | - missingfields = [f for f in computedfields if f not in lowercaseimportedfields] | |
| 240 | - missingemptyfields = [ f for f in missingfields if computedfields[f] == None or computedfields[f]['value'] == ''] | |
| 241 | - missingdefaultedfields = [ f for f in missingfields if f not in missingemptyfields] | |
| 242 | - | |
| 243 | - if missingfields != []: | |
| 244 | - processor.process_missing_fields(missingfields, missingemptyfields, missingdefaultedfields, computedfields) | |
| 245 | - | |
| 246 | - # end TODO: this is too complex | |
| 247 | - if notimportedfields != []: | |
| 248 | - processor.process_notimported_fields(notimportedfields) | |
| 249 | - | |
| 250 | - # TODO: test the cases where those fields have empty values. They should be handled as None. (just to test, may be working already :) | |
| 251 | - selects = [ | |
| 252 | - #Those ones inherit from AbstractEnum | |
| 253 | - ('type', model.Type), | |
| 254 | - ('status', model.Status), | |
| 255 | - ('priority', model.Priority), | |
| 256 | - ('severity', model.Severity), | |
| 257 | - ('resolution', model.Resolution), | |
| 258 | - #Those don't | |
| 259 | - ('milestone', model.Milestone), | |
| 260 | - ('component', model.Component), | |
| 261 | - ('version', model.Version) | |
| 262 | - ] | |
| 263 | - existingvalues = {} | |
| 264 | - newvalues = {} | |
| 265 | - for name, cls in selects: | |
| 266 | - if name not in lowercaseimportedfields: | |
| 267 | - # this field is not present, nothing to do | |
| 268 | - continue | |
| 269 | - | |
| 270 | - options = [val.name for val in cls.select(self.env)] | |
| 271 | - if not options: | |
| 272 | - # Fields without possible values are treated as if they didn't | |
| 273 | - # exist | |
| 274 | - continue | |
| 275 | - existingvalues[name] = options | |
| 276 | - newvalues[name] = [] | |
| 277 | - | |
| 278 | - | |
| 279 | - def add_sql_result(db, sql, list): | |
| 280 | - cursor = db.cursor() | |
| 281 | - cursor.execute(sql) | |
| 282 | - for result in cursor: | |
| 283 | - list += [ result ] | |
| 284 | - | |
| 285 | - | |
| 286 | - existingusers = [] | |
| 287 | - db = self.env.get_db_cnx() | |
| 288 | - add_sql_result(db, "SELECT DISTINCT reporter FROM ticket", existingusers) | |
| 289 | - add_sql_result(db, "SELECT DISTINCT owner FROM ticket", existingusers) | |
| 290 | - add_sql_result(db, "SELECT DISTINCT owner FROM component", existingusers) | |
| 291 | - | |
| 292 | - newusers = [] | |
| 293 | - | |
| 294 | - duplicate_summaries = [] | |
| 295 | - | |
| 296 | - row_idx = 0 | |
| 297 | - | |
| 298 | - for row in rows: | |
| 299 | - if idcolumn: | |
| 300 | - ticket_id = row[idcolumn] | |
| 301 | - if ticket_id: | |
| 302 | - self._check_ticket(db, ticket_id) | |
| 303 | - else: | |
| 304 | - # will create a new ticket | |
| 305 | - ticket_id = 0 | |
| 306 | - else: | |
| 307 | - summary = row[summarycolumn] | |
| 308 | - owner = ownercolumn and row[ownercolumn] or None | |
| 309 | - if self._skip_lines_with_empty_owner() and ownercolumn and not owner: | |
| 310 | - continue | |
| 311 | - | |
| 312 | - ticket_id = self._find_ticket(db, summary, owner) | |
| 313 | - if (summary, owner) in duplicate_summaries: | |
| 314 | - if owner == None: | |
| 315 | - raise TracError, 'Summary "%s" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % summary | |
| 316 | - else: | |
| 317 | - raise TracError, 'Summary "%s" and owner "%s" are duplicated in the spreadsheet. Ticket reconciliation can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % (summary, owner) | |
| 318 | - | |
| 319 | - else: | |
| 320 | - duplicate_summaries += [ (summary, owner) ] | |
| 321 | - | |
| 322 | - | |
| 323 | - processor.start_process_row(row_idx, ticket_id) | |
| 324 | - | |
| 325 | - for column in importedfields: | |
| 326 | - cell = row[column] | |
| 327 | - | |
| 328 | - # collect the new lookup values | |
| 329 | - if column.lower() in existingvalues.keys(): | |
| 330 | - if cell != '' and cell not in existingvalues[column.lower()] and cell not in newvalues[column.lower()]: | |
| 331 | - newvalues[column.lower()] += [ cell ] | |
| 332 | - | |
| 333 | - # also collect the new user names | |
| 334 | - if (column.lower() == 'owner' or column.lower() == 'reporter'): | |
| 335 | - if cell != '' and cell not in newusers and cell not in existingusers: | |
| 336 | - newusers += [ cell ] | |
| 337 | - | |
| 338 | - # and proces the value. | |
| 339 | - if column.lower() != 'ticket' and column.lower() != 'id': | |
| 340 | - processor.process_cell(column, cell) | |
| 341 | - | |
| 342 | - processor.end_process_row() | |
| 343 | - row_idx += 1 | |
| 344 | - | |
| 345 | - | |
| 346 | - if newvalues != {} and reduce(lambda x, y: x == [] and y or x, newvalues.values()) != []: | |
| 347 | - processor.process_new_lookups(newvalues) | |
| 348 | - | |
| 349 | - if newusers != []: | |
| 350 | - processor.process_new_users(newusers) | |
| 351 | - | |
| 352 | - return processor.end_process(row_idx) | |
| 353 | - | |
| 354 | - | |
| 355 | - def _reconciliate_by_owner_also(self): | |
| 356 | - return self.config.getbool('importer', 'reconciliate_by_owner_also', False) | |
| 357 | - | |
| 358 | - def _skip_lines_with_empty_owner(self): | |
| 359 | - return self.config.getbool('importer', 'skip_lines_with_empty_owner', False) | |
| 360 | - | |
| 361 | - def _find_case_insensitive(self, value, list): | |
| 362 | - ''' | |
| 363 | - Find case-insentively; returns the last (i.e. random !) element if not found | |
| 364 | - | |
| 365 | - >>> from trac.env import Environment | |
| 366 | - >>> import os | |
| 367 | - >>> instancedir = os.path.join(tempfile.gettempdir(), 'test-importer._find_case_insensitive') | |
| 368 | - >>> for root, dirs, files in os.walk(instancedir, topdown=False): | |
| 369 | - ... for name in files: | |
| 370 | - ... os.remove(os.path.join(root, name)) | |
| 371 | - ... for name in dirs: | |
| 372 | - ... os.rmdir(os.path.join(root, name)) | |
| 373 | - ... | |
| 374 | - >>> env = Environment(instancedir, create=True) | |
| 375 | - >>> importmodule = ImportModule(env) | |
| 376 | - >>> importmodule._find_case_insensitive('aa', ['Aa', 'Bb', 'Cc']) | |
| 377 | - 'Aa' | |
| 378 | - >>> importmodule._find_case_insensitive('aa', ['Cc', 'Aa', 'Bb']) | |
| 379 | - 'Aa' | |
| 380 | - >>> importmodule._find_case_insensitive('aa', ['Cc', 'Bb', 'Aa']) | |
| 381 | - 'Aa' | |
| 382 | - >>> importmodule._find_case_insensitive('dd', ['Aa', 'Cc', 'Bb']) | |
| 383 | - ''' | |
| 384 | - found = reduce(lambda x, y: ((y.lower() == value.lower()) and y or x), list) | |
| 385 | - return ((found.lower() == value.lower()) and found or None) | |
| 386 | - | |
| 387 | - def _find_ticket(self, db, summary, owner = None): | |
| 388 | - ''' | |
| 389 | - Finds the ticket(s) with the given summary | |
| 390 | - | |
| 391 | - >>> from trac.env import Environment | |
| 392 | - >>> import os | |
| 393 | - >>> instancedir = os.path.join(tempfile.gettempdir(), 'test-importer._find_ticket') | |
| 394 | - >>> for root, dirs, files in os.walk(instancedir, topdown=False): | |
| 395 | - ... for name in files: | |
| 396 | - ... os.remove(os.path.join(root, name)) | |
| 397 | - ... for name in dirs: | |
| 398 | - ... os.rmdir(os.path.join(root, name)) | |
| 399 | - ... | |
| 400 | - >>> env = Environment(instancedir, create=True) | |
| 401 | - >>> db = env.get_db_cnx() | |
| 402 | - >>> cursor = db.cursor() | |
| 403 | - >>> importmodule = ImportModule(env) | |
| 404 | - >>> def _exec(cursor, sql, args = None): cursor.execute(sql, args) | |
| 405 | - >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1235, 'AAAA']) | |
| 406 | - >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1236, "AA\'AA"]) | |
| 407 | - >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1237, "BBBB"]) | |
| 408 | - >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1238, "BBBB"]) | |
| 409 | - >>> db.commit() | |
| 410 | - >>> importmodule._find_ticket(db, 'AAAA') | |
| 411 | - 1235 | |
| 412 | - >>> importmodule._find_ticket(db, "AA\'AA") | |
| 413 | - 1236 | |
| 414 | - >>> importmodule._find_ticket(db, 'AA') | |
| 415 | - 0 | |
| 416 | - >>> try: importmodule._find_ticket(db, 'BBBB') | |
| 417 | - ... except TracError, err_string: print err_string | |
| 418 | - Tickets #1237 and #1238 have the same summary "BBBB" in Trac. Ticket reconciliation by summary can not be done. Please modify the summaries to ensure that they are unique. | |
| 419 | - >>> _exec(cursor, "delete from ticket where ID in (1235, 1236, 1237, 1238)") | |
| 420 | - >>> db.commit() | |
| 421 | - >>> #_exec(cursor, "select ticket,time,author,field,oldvalue,newvalue from ticket_change where time > 1190603709") | |
| 422 | - >>> _exec(cursor, "delete from ticket where id = 489") | |
| 423 | - >>> db.commit() | |
| 424 | - >>> #print cursor.fetchall() | |
| 425 | - >>> #importmodule._find_ticket(db, u'clusterization') | |
| 426 | - ''' | |
| 427 | - cursor = db.cursor() | |
| 428 | - if owner == None: | |
| 429 | - cursor.execute('SELECT id FROM ticket WHERE summary = %s', [ summary ] ) | |
| 430 | - else: | |
| 431 | - cursor.execute('SELECT id FROM ticket WHERE summary = %s and owner = %s', [ summary, owner ] ) | |
| 432 | - rows = cursor.fetchall() | |
| 433 | - if len(rows) > 1: | |
| 434 | - raise TracError('Tickets %s and %s have the same summary "%s" in Trac. Ticket reconciliation by summary can not be done. Please modify the summaries to ensure that they are unique.' % (reduce(lambda x, y: x + ', ' + y, ['#' + str(row[0]) for row in rows[0:-1]]), '#' + str(rows[-1][0]), summary)) | |
| 435 | - elif len(rows) == 1: | |
| 436 | - return int(rows[0][0]) | |
| 437 | - else: | |
| 438 | - return 0 | |
| 439 | - | |
| 440 | - def _check_ticket(self, db, ticket_id): | |
| 441 | - cursor = db.cursor() | |
| 442 | - | |
| 443 | - cursor.execute('SELECT summary FROM ticket WHERE id = %s', (str(ticket_id),)) | |
| 444 | - row = cursor.fetchone() | |
| 445 | - if not row: | |
| 446 | - raise TracError('Ticket %s found in file, but not present in Trac: cannot import.' % str(ticket_id)) | |
| 447 | - return row[0] | |
| 448 | - | |
| 449 | -if __name__ == '__main__': | |
| 450 | - import doctest | |
| 451 | - testfolder = __file__ | |
| 452 | - doctest.testmod() | |
| 1 | +# -*- coding: utf-8 -*- | |
| 2 | +# | |
| 3 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 4 | +# Author: Francois Granade - fg at nexb dot com | |
| 5 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 6 | +# | |
| 7 | + | |
| 8 | +import os | |
| 9 | +import re | |
| 10 | +import shutil | |
| 11 | +import tempfile | |
| 12 | +import time | |
| 13 | +import unicodedata | |
| 14 | + | |
| 15 | +from trac.core import * | |
| 16 | +from trac.attachment import AttachmentModule | |
| 17 | +from trac.core import Component | |
| 18 | +from trac.perm import IPermissionRequestor | |
| 19 | +from trac.ticket import TicketSystem | |
| 20 | +from trac.ticket import model | |
| 21 | +from trac.util import get_reporter_id | |
| 22 | +from trac.util.html import html | |
| 23 | +from trac.web import IRequestHandler | |
| 24 | +from trac.web.chrome import INavigationContributor, ITemplateProvider | |
| 25 | + | |
| 26 | +from talm_importer.processors import ImportProcessor | |
| 27 | +from talm_importer.processors import PreviewProcessor | |
| 28 | +from talm_importer.readers import get_reader | |
| 29 | + | |
| 30 | + | |
| 31 | +class ImportModule(Component): | |
| 32 | + | |
| 33 | + implements(INavigationContributor, IPermissionRequestor, IRequestHandler, ITemplateProvider) | |
| 34 | + | |
| 35 | + # INavigationContributor methods | |
| 36 | + def get_active_navigation_item(self, req): | |
| 37 | + return 'importer' | |
| 38 | + | |
| 39 | + def get_navigation_items(self, req): | |
| 40 | + if not req.perm.has_permission('IMPORT_EXECUTE'): | |
| 41 | + return | |
| 42 | + yield ('mainnav', 'importer', | |
| 43 | + html.a('Excelｲﾝﾎﾟｰﾄ', href=req.href.importer())) | |
| 44 | + | |
| 45 | + # IPermissionRequestor methods | |
| 46 | + def get_permission_actions(self): | |
| 47 | + return ['IMPORT_EXECUTE'] | |
| 48 | + | |
| 49 | + # IRequestHandler methods | |
| 50 | + | |
| 51 | + def match_request(self, req): | |
| 52 | + match = re.match(r'/importer(?:/([0-9]+))?', req.path_info) | |
| 53 | + if match: | |
| 54 | + return True | |
| 55 | + | |
| 56 | + def process_request(self, req): | |
| 57 | + req.perm.assert_permission('IMPORT_EXECUTE') | |
| 58 | + action = req.args.get('action', 'other') | |
| 59 | + | |
| 60 | + if req.args.has_key('cancel'): | |
| 61 | + req.redirect(req.href('importer')) | |
| 62 | + | |
| 63 | + if action == 'upload' and req.method == 'POST': | |
| 64 | + req.session['uploadedfile'] = None | |
| 65 | + uploadedfilename, uploadedfile = self._save_uploaded_file(req) | |
| 66 | + req.session['sheet'] = req.args['sheet'] | |
| 67 | + req.session['uploadedfile'] = uploadedfile | |
| 68 | + req.session['uploadedfilename'] = uploadedfilename | |
| 69 | + req.session['tickettime'] = str(int(time.time())) | |
| 70 | + return self._do_preview(uploadedfile, int(req.session['sheet']), req) | |
| 71 | + elif action == 'import' and req.method == 'POST': | |
| 72 | + tickettime = int(req.session['tickettime']) | |
| 73 | + if tickettime == 0: | |
| 74 | + raise TracError('No time set since preview, unable to import: please upload the file again') | |
| 75 | + | |
| 76 | + return self._do_import(req.session['uploadedfile'], int(req.session['sheet']), req, req.session['uploadedfilename'], tickettime) | |
| 77 | + | |
| 78 | + else: | |
| 79 | + req.session['uploadedfile'] = None | |
| 80 | + req.session['uploadedfilename'] = None | |
| 81 | + | |
| 82 | + data = { 'reconciliate_by_owner_also': self._reconciliate_by_owner_also(), | |
| 83 | + 'fields': ['ticket or id'] + [field['name'] for field in TicketSystem(self.env).get_ticket_fields()] } | |
| 84 | + | |
| 85 | + return 'importer.html', data, None | |
| 86 | + | |
| 87 | + # ITemplateProvider | |
| 88 | + | |
| 89 | + def get_htdocs_dirs(self): | |
| 90 | + """Return the absolute path of a directory containing additional | |
| 91 | + static resources (such as images, style sheets, etc). | |
| 92 | + """ | |
| 93 | + return [] | |
| 94 | + | |
| 95 | + def get_templates_dirs(self): | |
| 96 | + """Return the absolute path of the directory containing the provided | |
| 97 | + ClearSilver templates. | |
| 98 | + """ | |
| 99 | + from pkg_resources import resource_filename | |
| 100 | + return [resource_filename(__name__, 'templates')] | |
| 101 | + | |
| 102 | + # Internal methods | |
| 103 | + | |
| 104 | + def _do_preview(self, uploadedfile, sheet, req): | |
| 105 | + filereader = get_reader(uploadedfile, sheet) | |
| 106 | + try: | |
| 107 | + return self._process(filereader, get_reporter_id(req), PreviewProcessor(self.env, req)) | |
| 108 | + finally: | |
| 109 | + filereader.close() | |
| 110 | + | |
| 111 | + def _do_import(self, uploadedfile, sheet, req, uploadedfilename, tickettime): | |
| 112 | + filereader = get_reader(uploadedfile, sheet) | |
| 113 | + try: | |
| 114 | + try: | |
| 115 | + return self._process(filereader, get_reporter_id(req), ImportProcessor(self.env, req, uploadedfilename, tickettime)) | |
| 116 | + finally: | |
| 117 | + filereader.close() | |
| 118 | + except: | |
| 119 | + # Unlock the database. This is not really tested, but seems reasonable. TODO: test or verify this | |
| 120 | + self.env.get_db_cnx().rollback() | |
| 121 | + raise | |
| 122 | + | |
| 123 | + | |
| 124 | + def _save_uploaded_file(self, req): | |
| 125 | + req.perm.assert_permission('IMPORT_EXECUTE') | |
| 126 | + | |
| 127 | + upload = req.args['import-file'] | |
| 128 | + if not hasattr(upload, 'filename') or not upload.filename: | |
| 129 | + raise TracError('No file uploaded') | |
| 130 | + if hasattr(upload.file, 'fileno'): | |
| 131 | + size = os.fstat(upload.file.fileno())[6] | |
| 132 | + else: | |
| 133 | + upload.file.seek(0, 2) # seek to end of file | |
| 134 | + size = upload.file.tell() | |
| 135 | + upload.file.seek(0) | |
| 136 | + if size == 0: | |
| 137 | + raise TracError("Can't upload empty file") | |
| 138 | + | |
| 139 | + # Maximum file size (in bytes) | |
| 140 | + max_size = AttachmentModule.max_size | |
| 141 | + if max_size >= 0 and size > max_size: | |
| 142 | + raise TracError('Maximum file size (same as attachment size, set in trac.ini configuration file): %d bytes' % max_size, | |
| 143 | + 'Upload failed') | |
| 144 | + | |
| 145 | + # We try to normalize the filename to unicode NFC if we can. | |
| 146 | + # Files uploaded from OS X might be in NFD. | |
| 147 | + filename = unicodedata.normalize('NFC', unicode(upload.filename, | |
| 148 | + 'utf-8')) | |
| 149 | + filename = filename.replace('\\', '/').replace(':', '/') | |
| 150 | + filename = os.path.basename(filename) | |
| 151 | + if not filename: | |
| 152 | + raise TracError('No file uploaded') | |
| 153 | + | |
| 154 | + return filename, self._savedata(upload.file) | |
| 155 | + | |
| 156 | + | |
| 157 | + def _savedata(self, fileobj): | |
| 158 | + | |
| 159 | + # temp folder | |
| 160 | + tempuploadedfile = tempfile.mktemp() | |
| 161 | + | |
| 162 | + flags = os.O_CREAT + os.O_WRONLY + os.O_EXCL | |
| 163 | + if hasattr(os, 'O_BINARY'): | |
| 164 | + flags += os.O_BINARY | |
| 165 | + targetfile = os.fdopen(os.open(tempuploadedfile, flags), 'w') | |
| 166 | + | |
| 167 | + try: | |
| 168 | + shutil.copyfileobj(fileobj, targetfile) | |
| 169 | + finally: | |
| 170 | + targetfile.close() | |
| 171 | + return tempuploadedfile | |
| 172 | + | |
| 173 | + | |
| 174 | + | |
| 175 | + def _process(self, filereader, reporter, processor): | |
| 176 | + tracfields = [field['name'] for field in TicketSystem(self.env).get_ticket_fields()] | |
| 177 | + tracfields = [ 'ticket', 'id' ] + tracfields | |
| 178 | + customfields = [field['name'] for field in TicketSystem(self.env).get_custom_fields()] | |
| 179 | + | |
| 180 | + columns, rows = filereader.readers() | |
| 181 | + | |
| 182 | + # defensive: columns could be non-string, make sure they are | |
| 183 | + columns = map(str, columns) | |
| 184 | + | |
| 185 | + importedfields = [f for f in columns if f.lower() in tracfields] | |
| 186 | + notimportedfields = [f for f in columns if f.lower() not in tracfields + ['comment']] | |
| 187 | + commentfields = [f for f in columns if f.lower() == 'comment'] | |
| 188 | + if commentfields: | |
| 189 | + commentfield = commentfields[0] | |
| 190 | + else: | |
| 191 | + commentfield = None | |
| 192 | + lowercaseimportedfields = [f.lower() for f in importedfields] | |
| 193 | + | |
| 194 | + idcolumn = None | |
| 195 | + | |
| 196 | + if 'ticket' in lowercaseimportedfields and 'id' in lowercaseimportedfields: | |
| 197 | + raise TracError, 'The first line of the worksheet contains both \'ticket\', and an \'id\' field name. Only one of them is needed to perform the import. Please check the file and try again.' | |
| 198 | + | |
| 199 | + ownercolumn = None | |
| 200 | + if 'ticket' in lowercaseimportedfields: | |
| 201 | + idcolumn = self._find_case_insensitive('ticket', importedfields) | |
| 202 | + elif 'id' in lowercaseimportedfields: | |
| 203 | + idcolumn = self._find_case_insensitive('id', importedfields) | |
| 204 | + elif 'summary' in lowercaseimportedfields: | |
| 205 | + summarycolumn = self._find_case_insensitive('summary', importedfields) | |
| 206 | + ownercolumn = self._reconciliate_by_owner_also() and self._find_case_insensitive('owner', importedfields) or None | |
| 207 | + else: | |
| 208 | + raise TracError, 'The first line of the worksheet contains neither a \'ticket\', an \'id\' nor a \'summary\' field name. At least one of them is needed to perform the import. Please check the file and try again.' | |
| 209 | + | |
| 210 | + # start TODO: this is too complex, it should be replaced by a call to TicketSystem(env).get_ticket_fields() | |
| 211 | + | |
| 212 | + # The fields that we will have to set a value for, if: | |
| 213 | + # - they are not in the imported fields, and | |
| 214 | + # - they are not set in the default values of the Ticket class, and | |
| 215 | + # - they shouldn't be set to empty | |
| 216 | + # if 'set' is true, this will be the value that will be set by default (even if the default value in the Ticket class is different) | |
| 217 | + # if 'set' is false, the value is computed by Trac and we don't have anything to do | |
| 218 | + computedfields = {'status': { 'value':'new', 'set': True }, | |
| 219 | + 'resolution' : { 'value': "''(None)''", 'set': False }, | |
| 220 | + 'reporter' : { 'value': reporter, 'set': True }, | |
| 221 | + 'time' : { 'value': "''(now)''", 'set': False }, | |
| 222 | + 'changetime' : { 'value': "''(now)''", 'set': False } } | |
| 223 | + | |
| 224 | + if 'owner' not in lowercaseimportedfields and 'component' in lowercaseimportedfields: | |
| 225 | + computedfields['owner'] = {} | |
| 226 | + computedfields['owner']['value'] = 'Computed from component' | |
| 227 | + computedfields['owner']['set'] = False | |
| 228 | + | |
| 229 | + # to get the compulted default values | |
| 230 | + from ticket import PatchedTicket | |
| 231 | + ticket = PatchedTicket(self.env) | |
| 232 | + | |
| 233 | + for f in [ 'type', 'cc' , 'url', 'description', 'keywords', 'component' , 'severity' , 'priority' , 'version', 'milestone' ] + customfields: | |
| 234 | + if f in ticket.values: | |
| 235 | + computedfields[f] = {} | |
| 236 | + computedfields[f]['value'] = ticket.values[f] | |
| 237 | + computedfields[f]['set'] = False | |
| 238 | + else: | |
| 239 | + computedfields[f] = None | |
| 240 | + | |
| 241 | + processor.start(importedfields, ownercolumn != None, commentfield) | |
| 242 | + | |
| 243 | + missingfields = [f for f in computedfields if f not in lowercaseimportedfields] | |
| 244 | + missingemptyfields = [ f for f in missingfields if computedfields[f] == None or computedfields[f]['value'] == ''] | |
| 245 | + missingdefaultedfields = [ f for f in missingfields if f not in missingemptyfields] | |
| 246 | + | |
| 247 | + if missingfields != []: | |
| 248 | + processor.process_missing_fields(missingfields, missingemptyfields, missingdefaultedfields, computedfields) | |
| 249 | + | |
| 250 | + # end TODO: this is too complex | |
| 251 | + if notimportedfields != []: | |
| 252 | + processor.process_notimported_fields(notimportedfields) | |
| 253 | + | |
| 254 | + if commentfield: | |
| 255 | + processor.process_comment_field(commentfield) | |
| 256 | + | |
| 257 | + # TODO: test the cases where those fields have empty values. They should be handled as None. (just to test, may be working already :) | |
| 258 | + selects = [ | |
| 259 | + #Those ones inherit from AbstractEnum | |
| 260 | + ('type', model.Type), | |
| 261 | + ('status', model.Status), | |
| 262 | + ('priority', model.Priority), | |
| 263 | + ('severity', model.Severity), | |
| 264 | + ('resolution', model.Resolution), | |
| 265 | + #Those don't | |
| 266 | + ('milestone', model.Milestone), | |
| 267 | + ('component', model.Component), | |
| 268 | + ('version', model.Version) | |
| 269 | + ] | |
| 270 | + existingvalues = {} | |
| 271 | + newvalues = {} | |
| 272 | + for name, cls in selects: | |
| 273 | + if name not in lowercaseimportedfields: | |
| 274 | + # this field is not present, nothing to do | |
| 275 | + continue | |
| 276 | + | |
| 277 | + options = [val.name for val in cls.select(self.env)] | |
| 278 | + if not options: | |
| 279 | + # Fields without possible values are treated as if they didn't | |
| 280 | + # exist | |
| 281 | + continue | |
| 282 | + existingvalues[name] = options | |
| 283 | + newvalues[name] = [] | |
| 284 | + | |
| 285 | + | |
| 286 | + def add_sql_result(db, sql, list): | |
| 287 | + cursor = db.cursor() | |
| 288 | + cursor.execute(sql) | |
| 289 | + for result in cursor: | |
| 290 | + list += [ result ] | |
| 291 | + | |
| 292 | + | |
| 293 | + existingusers = [] | |
| 294 | + db = self.env.get_db_cnx() | |
| 295 | + add_sql_result(db, "SELECT DISTINCT reporter FROM ticket", existingusers) | |
| 296 | + add_sql_result(db, "SELECT DISTINCT owner FROM ticket", existingusers) | |
| 297 | + add_sql_result(db, "SELECT DISTINCT owner FROM component", existingusers) | |
| 298 | + | |
| 299 | + newusers = [] | |
| 300 | + | |
| 301 | + duplicate_summaries = [] | |
| 302 | + | |
| 303 | + row_idx = 0 | |
| 304 | + indent = 0 | |
| 305 | + | |
| 306 | + for row in rows: | |
| 307 | + if idcolumn: | |
| 308 | + ticket_id = row[idcolumn] | |
| 309 | + if ticket_id: | |
| 310 | + self._check_ticket(db, ticket_id) | |
| 311 | + else: | |
| 312 | + # will create a new ticket | |
| 313 | + ticket_id = 0 | |
| 314 | + else: | |
| 315 | + summary = row[summarycolumn] | |
| 316 | + owner = ownercolumn and row[ownercolumn] or None | |
| 317 | + if self._skip_lines_with_empty_owner() and ownercolumn and not owner: | |
| 318 | + continue | |
| 319 | + | |
| 320 | + ticket_id = self._find_ticket(db, summary, owner) | |
| 321 | + if (summary, owner) in duplicate_summaries: | |
| 322 | + if owner == None: | |
| 323 | + raise TracError, 'Summary "%s" is duplicated in the spreadsheet. Ticket reconciliation by summary can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % summary | |
| 324 | + else: | |
| 325 | + raise TracError, 'Summary "%s" and owner "%s" are duplicated in the spreadsheet. Ticket reconciliation can not be done. Please modify the summaries in the spreadsheet to ensure that they are unique.' % (summary, owner) | |
| 326 | + | |
| 327 | + else: | |
| 328 | + duplicate_summaries += [ (summary, owner) ] | |
| 329 | + | |
| 330 | + | |
| 331 | + processor.start_process_row(row_idx, ticket_id) | |
| 332 | + | |
| 333 | + for column in importedfields: | |
| 334 | + cell = row[column] | |
| 335 | + | |
| 336 | + # calc indent if column is summary | |
| 337 | + if column.lower() == 'summary': | |
| 338 | + indent = len(cell)-len(cell.lstrip()) | |
| 339 | + | |
| 340 | + # collect the new lookup values | |
| 341 | + if column.lower() in existingvalues.keys(): | |
| 342 | + cell = cell.strip() | |
| 343 | + if cell != '' and cell not in existingvalues[column.lower()] and cell not in newvalues[column.lower()]: | |
| 344 | + newvalues[column.lower()] += [ cell ] | |
| 345 | + | |
| 346 | + # also collect the new user names | |
| 347 | + if (column.lower() == 'owner' or column.lower() == 'reporter'): | |
| 348 | + if cell != '' and cell not in newusers and cell not in existingusers: | |
| 349 | + newusers += [ cell ] | |
| 350 | + | |
| 351 | + # and proces the value. | |
| 352 | + if column.lower() != 'ticket' and column.lower() != 'id': | |
| 353 | + processor.process_cell(column, cell) | |
| 354 | + | |
| 355 | + if commentfield: | |
| 356 | + processor.process_comment(row[commentfield]) | |
| 357 | + | |
| 358 | + processor.end_process_row(indent) | |
| 359 | + row_idx += 1 | |
| 360 | + | |
| 361 | + | |
| 362 | + if newvalues != {} and reduce(lambda x, y: x == [] and y or x, newvalues.values()) != []: | |
| 363 | + processor.process_new_lookups(newvalues) | |
| 364 | + | |
| 365 | + if newusers != []: | |
| 366 | + processor.process_new_users(newusers) | |
| 367 | + | |
| 368 | + return processor.end_process(row_idx) | |
| 369 | + | |
| 370 | + | |
| 371 | + def _reconciliate_by_owner_also(self): | |
| 372 | + return self.config.getbool('importer', 'reconciliate_by_owner_also', False) | |
| 373 | + | |
| 374 | + def _skip_lines_with_empty_owner(self): | |
| 375 | + return self.config.getbool('importer', 'skip_lines_with_empty_owner', False) | |
| 376 | + | |
| 377 | + def _find_case_insensitive(self, value, list): | |
| 378 | + ''' | |
| 379 | + Find case-insentively; returns the last (i.e. random !) element if not found | |
| 380 | + | |
| 381 | + >>> from trac.env import Environment | |
| 382 | + >>> import os | |
| 383 | + >>> instancedir = os.path.join(tempfile.gettempdir(), 'test-importer._find_case_insensitive') | |
| 384 | + >>> for root, dirs, files in os.walk(instancedir, topdown=False): | |
| 385 | + ... for name in files: | |
| 386 | + ... os.remove(os.path.join(root, name)) | |
| 387 | + ... for name in dirs: | |
| 388 | + ... os.rmdir(os.path.join(root, name)) | |
| 389 | + ... | |
| 390 | + >>> env = Environment(instancedir, create=True) | |
| 391 | + >>> importmodule = ImportModule(env) | |
| 392 | + >>> importmodule._find_case_insensitive('aa', ['Aa', 'Bb', 'Cc']) | |
| 393 | + 'Aa' | |
| 394 | + >>> importmodule._find_case_insensitive('aa', ['Cc', 'Aa', 'Bb']) | |
| 395 | + 'Aa' | |
| 396 | + >>> importmodule._find_case_insensitive('aa', ['Cc', 'Bb', 'Aa']) | |
| 397 | + 'Aa' | |
| 398 | + >>> importmodule._find_case_insensitive('dd', ['Aa', 'Cc', 'Bb']) | |
| 399 | + ''' | |
| 400 | + found = reduce(lambda x, y: ((y.lower() == value.lower()) and y or x), list) | |
| 401 | + return ((found.lower() == value.lower()) and found or None) | |
| 402 | + | |
| 403 | + def _find_ticket(self, db, summary, owner = None): | |
| 404 | + ''' | |
| 405 | + Finds the ticket(s) with the given summary | |
| 406 | + | |
| 407 | + >>> from trac.env import Environment | |
| 408 | + >>> import os | |
| 409 | + >>> instancedir = os.path.join(tempfile.gettempdir(), 'test-importer._find_ticket') | |
| 410 | + >>> for root, dirs, files in os.walk(instancedir, topdown=False): | |
| 411 | + ... for name in files: | |
| 412 | + ... os.remove(os.path.join(root, name)) | |
| 413 | + ... for name in dirs: | |
| 414 | + ... os.rmdir(os.path.join(root, name)) | |
| 415 | + ... | |
| 416 | + >>> env = Environment(instancedir, create=True) | |
| 417 | + >>> db = env.get_db_cnx() | |
| 418 | + >>> cursor = db.cursor() | |
| 419 | + >>> importmodule = ImportModule(env) | |
| 420 | + >>> def _exec(cursor, sql, args = None): cursor.execute(sql, args) | |
| 421 | + >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1235, 'AAAA']) | |
| 422 | + >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1236, "AA\'AA"]) | |
| 423 | + >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1237, "BBBB"]) | |
| 424 | + >>> _exec(cursor, "insert into ticket (id, summary) values (%s, %s)", [1238, "BBBB"]) | |
| 425 | + >>> db.commit() | |
| 426 | + >>> importmodule._find_ticket(db, 'AAAA') | |
| 427 | + 1235 | |
| 428 | + >>> importmodule._find_ticket(db, "AA\'AA") | |
| 429 | + 1236 | |
| 430 | + >>> importmodule._find_ticket(db, 'AA') | |
| 431 | + 0 | |
| 432 | + >>> try: importmodule._find_ticket(db, 'BBBB') | |
| 433 | + ... except TracError, err_string: print err_string | |
| 434 | + Tickets #1237 and #1238 have the same summary "BBBB" in Trac. Ticket reconciliation by summary can not be done. Please modify the summaries to ensure that they are unique. | |
| 435 | + >>> _exec(cursor, "delete from ticket where ID in (1235, 1236, 1237, 1238)") | |
| 436 | + >>> db.commit() | |
| 437 | + >>> #_exec(cursor, "select ticket,time,author,field,oldvalue,newvalue from ticket_change where time > 1190603709") | |
| 438 | + >>> _exec(cursor, "delete from ticket where id = 489") | |
| 439 | + >>> db.commit() | |
| 440 | + >>> #print cursor.fetchall() | |
| 441 | + >>> #importmodule._find_ticket(db, u'clusterization') | |
| 442 | + ''' | |
| 443 | + cursor = db.cursor() | |
| 444 | + if owner == None: | |
| 445 | + cursor.execute('SELECT id FROM ticket WHERE summary = %s', [ summary ] ) | |
| 446 | + else: | |
| 447 | + cursor.execute('SELECT id FROM ticket WHERE summary = %s and owner = %s', [ summary, owner ] ) | |
| 448 | + rows = cursor.fetchall() | |
| 449 | + if len(rows) > 1: | |
| 450 | + raise TracError('Tickets %s and %s have the same summary "%s" in Trac. Ticket reconciliation by summary can not be done. Please modify the summaries to ensure that they are unique.' % (reduce(lambda x, y: x + ', ' + y, ['#' + str(row[0]) for row in rows[0:-1]]), '#' + str(rows[-1][0]), summary)) | |
| 451 | + elif len(rows) == 1: | |
| 452 | + return int(rows[0][0]) | |
| 453 | + else: | |
| 454 | + return 0 | |
| 455 | + | |
| 456 | + def _check_ticket(self, db, ticket_id): | |
| 457 | + cursor = db.cursor() | |
| 458 | + | |
| 459 | + cursor.execute('SELECT summary FROM ticket WHERE id = %s', (str(ticket_id),)) | |
| 460 | + row = cursor.fetchone() | |
| 461 | + if not row: | |
| 462 | + raise TracError('Ticket %s found in file, but not present in Trac: cannot import.' % str(ticket_id)) | |
| 463 | + return row[0] | |
| 464 | + | |
| 465 | +if __name__ == '__main__': | |
| 466 | + import doctest | |
| 467 | + testfolder = __file__ | |
| 468 | + doctest.testmod() |
| @@ -1,254 +1,292 @@ | ||
| 1 | -# | |
| 2 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | -# Author: Francois Granade - fg at nexb dot com | |
| 4 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | -# | |
| 6 | - | |
| 7 | -import time | |
| 8 | - | |
| 9 | -from trac.ticket import Ticket, model | |
| 10 | -from trac.util import get_reporter_id | |
| 11 | -from trac.util.datefmt import format_datetime | |
| 12 | -from trac.util.html import Markup | |
| 13 | -from trac.wiki import wiki_to_html | |
| 14 | - | |
| 15 | - | |
| 16 | -class ImportProcessor(object): | |
| 17 | - def __init__(self, env, req, filename, tickettime): | |
| 18 | - self.env = env | |
| 19 | - self.req = req | |
| 20 | - self.filename = filename | |
| 21 | - self.modifiedcount = 0 | |
| 22 | - self.notmodifiedcount = 0 | |
| 23 | - self.added = 0 | |
| 24 | - | |
| 25 | - # TODO: check that the tickets haven't changed since preview | |
| 26 | - self.tickettime = tickettime | |
| 27 | - | |
| 28 | - # Keep the db to commit it all at once at the end | |
| 29 | - self.db = self.env.get_db_cnx() | |
| 30 | - self.missingemptyfields = None | |
| 31 | - self.missingdefaultedfields = None | |
| 32 | - self.computedfields = None | |
| 33 | - | |
| 34 | - def start(self, importedfields, reconciliate_by_owner_also): | |
| 35 | - pass | |
| 36 | - | |
| 37 | - def process_missing_fields(self, missingfields, missingemptyfields, missingdefaultedfields, computedfields): | |
| 38 | - self.missingemptyfields = missingemptyfields | |
| 39 | - self.missingdefaultedfields = missingdefaultedfields | |
| 40 | - self.computedfields = computedfields | |
| 41 | - | |
| 42 | - def process_notimported_fields(self, notimportedfields): | |
| 43 | - pass | |
| 44 | - | |
| 45 | - def start_process_row(self, row_idx, ticket_id): | |
| 46 | - from ticket import PatchedTicket | |
| 47 | - if ticket_id > 0: | |
| 48 | - # existing ticket | |
| 49 | - self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db) | |
| 50 | - | |
| 51 | - # 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10. | |
| 52 | - # if we have trac.util.datefmt.to_datetime, we're likely with 0.11 | |
| 53 | - try: | |
| 54 | - from trac.util.datefmt import to_timestamp | |
| 55 | - time_changed = to_timestamp(self.ticket.time_changed) | |
| 56 | - except ImportError: | |
| 57 | - time_changed = int(self.ticket.time_changed) | |
| 58 | - | |
| 59 | - if time_changed > self.tickettime: | |
| 60 | - # just in case, verify if it wouldn't be a ticket that has been modified in the future | |
| 61 | - # (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error | |
| 62 | - if time_changed < int(time.time()): | |
| 63 | - # TODO: this is not working yet... | |
| 64 | - # | |
| 65 | - #raise TracError("Sorry, can not execute the import. " | |
| 66 | - #"The ticket #" + str(ticket_id) + " has been modified by someone else " | |
| 67 | - #"since preview. You must re-upload and preview your file to avoid overwriting the other changes.") | |
| 68 | - pass | |
| 69 | - | |
| 70 | - else: | |
| 71 | - self.ticket = PatchedTicket(self.env, db=self.db) | |
| 72 | - | |
| 73 | - def process_cell(self, column, cell): | |
| 74 | - cell = unicode(cell) | |
| 75 | - # this will ensure that the changes are logged, see model.py Ticket.__setitem__ | |
| 76 | - self.ticket[column.lower()] = cell | |
| 77 | - | |
| 78 | - def end_process_row(self): | |
| 79 | - try: | |
| 80 | - # 'when' is a datetime in 0.11, and an int in 0.10. | |
| 81 | - # if we have trac.util.datefmt.to_datetime, we're likely with 0.11 | |
| 82 | - from trac.util.datefmt import to_datetime | |
| 83 | - tickettime = to_datetime(self.tickettime) | |
| 84 | - except ImportError: | |
| 85 | - tickettime = self.tickettime | |
| 86 | - | |
| 87 | - if self.ticket.id == None: | |
| 88 | - for f in self.missingemptyfields: | |
| 89 | - if self.ticket.values.has_key(f) and self.ticket[f] == None: | |
| 90 | - self.ticket[f] = '' | |
| 91 | - for f in self.computedfields: | |
| 92 | - if self.computedfields[f] != None and self.computedfields[f]['set']: | |
| 93 | - self.ticket[f] = self.computedfields[f]['value'] | |
| 94 | - | |
| 95 | - self.added += 1 | |
| 96 | - self.ticket.insert(when=tickettime, db=self.db) | |
| 97 | - else: | |
| 98 | - message = "Batch update from file " + self.filename | |
| 99 | - if self.ticket.is_modified(): | |
| 100 | - self.modifiedcount += 1 | |
| 101 | - self.ticket.save_changes(get_reporter_id(self.req), message, when=tickettime, db=self.db) # TODO: handle cnum, cnum = ticket.values['cnum'] + 1) | |
| 102 | - else: | |
| 103 | - self.notmodifiedcount += 1 | |
| 104 | - | |
| 105 | - self.ticket = None | |
| 106 | - | |
| 107 | - def process_new_lookups(self, newvalues): | |
| 108 | - for field, names in newvalues.iteritems(): | |
| 109 | - if names == []: | |
| 110 | - continue | |
| 111 | - if field == 'component': | |
| 112 | - class CurrentLookupEnum(model.Component): | |
| 113 | - pass | |
| 114 | - elif field == 'milestone': | |
| 115 | - class CurrentLookupEnum(model.Milestone): | |
| 116 | - pass | |
| 117 | - elif field == 'version': | |
| 118 | - class CurrentLookupEnum(model.Version): | |
| 119 | - pass | |
| 120 | - else: | |
| 121 | - class CurrentLookupEnum(model.AbstractEnum): | |
| 122 | - # here, you shouldn't put 'self.' before the class field. | |
| 123 | - type = field | |
| 124 | - | |
| 125 | - for name in names: | |
| 126 | - lookup = CurrentLookupEnum(self.env, db=self.db) | |
| 127 | - lookup.name = name | |
| 128 | - lookup.insert() | |
| 129 | - | |
| 130 | - def process_new_users(self, newusers): | |
| 131 | - pass | |
| 132 | - | |
| 133 | - def end_process(self, numrows): | |
| 134 | - self.db.commit() | |
| 135 | - | |
| 136 | - data = {} | |
| 137 | - data['title'] = 'Import completed' | |
| 138 | - #data['report.title'] = data['title'].lower() | |
| 139 | - | |
| 140 | - message = 'Successfully imported ' + str(numrows) + ' tickets (' + str(self.added) + ' added, ' + str(self.modifiedcount) + ' modified, ' + str(self.notmodifiedcount) + ' unchanged).' | |
| 141 | - | |
| 142 | - data['message'] = Markup("<style type=\"text/css\">#report-notfound { display:none; }</style>\n") + wiki_to_html(message, self.env, self.req) | |
| 143 | - | |
| 144 | - return 'import_preview.html', data, None | |
| 145 | - | |
| 146 | - | |
| 147 | -class PreviewProcessor(object): | |
| 148 | - | |
| 149 | - def __init__(self, env, req): | |
| 150 | - self.env = env | |
| 151 | - self.req = req | |
| 152 | - self.data = {'rows': []} | |
| 153 | - self.ticket = None | |
| 154 | - self.modified = False | |
| 155 | - self.styles = '' | |
| 156 | - self.duplicatessumaries = [] | |
| 157 | - self.modifiedcount = 0 | |
| 158 | - self.notmodifiedcount = 0 | |
| 159 | - self.added = 0 | |
| 160 | - | |
| 161 | - def start(self, importedfields, reconciliate_by_owner_also): | |
| 162 | - self.data['title'] = 'Preview Import' | |
| 163 | - | |
| 164 | - self.message = '' | |
| 165 | - | |
| 166 | - if 'ticket' in [f.lower() for f in importedfields]: | |
| 167 | - self.message += ' * A \'\'\'ticket\'\'\' column was found: Existing tickets will be updated with the values from the file. Values that are changing appear in italics in the preview below.\n' | |
| 168 | - elif 'id' in [f.lower() for f in importedfields]: | |
| 169 | - self.message += ' * A \'\'\'id\'\'\' column was found: Existing tickets will be updated with the values from the file. Values that are changing appear in italics in the preview below.\n' | |
| 170 | - else: | |
| 171 | - self.message += ' * A \'\'\'ticket\'\'\' column was not found: tickets will be reconciliated by summary' + (reconciliate_by_owner_also and ' and by owner' or '') + '. If an existing ticket with the same summary' + (reconciliate_by_owner_also and ' and the same owner' or '') + ' is found, values that are changing appear in italics in the preview below. If no ticket with same summary ' + (reconciliate_by_owner_also and ' and same owner' or '') + 'is found, the whole line appears in italics below, and a new ticket will be added.\n' | |
| 172 | - | |
| 173 | - self.data['headers'] = [{ 'col': 'ticket', 'title': 'ticket' }] | |
| 174 | - # we use one more color to set a style for all fields in a row... the CS templates happens 'color' + color + '-odd' | |
| 175 | - self.styles = "<style type=\"text/css\">\n.ticket-imported, .modified-ticket-imported { width: 40px; }\n" | |
| 176 | - self.styles += ".color-new-odd td, .color-new-even td, .modified-ticket-imported" | |
| 177 | - for col in importedfields: | |
| 178 | - if col.lower() != 'ticket' and col.lower() != 'id': | |
| 179 | - title=col.capitalize() | |
| 180 | - self.data['headers'].append({ 'col': col, 'title': title }) | |
| 181 | - self.styles += ", .modified-%s" % col | |
| 182 | - self.styles += " { font-style: italic; }\n" | |
| 183 | - self.styles += "</style>\n" | |
| 184 | - | |
| 185 | - # This could be simplified... | |
| 186 | - def process_missing_fields(self, missingfields, missingemptyfields, missingdefaultedfields, computedfields): | |
| 187 | - self.message += ' * Some Trac fields are not present in the import. They will default to:\n\n' | |
| 188 | - self.message += " ||'''field'''||'''Default value'''||\n" | |
| 189 | - if missingemptyfields != []: | |
| 190 | - self.message += ' ||' + ', '.join([x.capitalize() for x in missingemptyfields]) + '||' + "''(Empty value)''" + '||\n' | |
| 191 | - | |
| 192 | - if missingdefaultedfields != []: | |
| 193 | - for f in missingdefaultedfields: | |
| 194 | - self.message += ' ||' + f.capitalize() + '||' + str(computedfields[f]['value']) + '||\n' | |
| 195 | - | |
| 196 | - self.message += '(You can change some of these default values in the Trac Admin module, if you are administrator; or you can add the corresponding column to your spreadsheet and re-upload it).\n' | |
| 197 | - | |
| 198 | - def process_notimported_fields(self, notimportedfields): | |
| 199 | - self.message += ' * Some fields will not be imported because they don\'t exist in Trac: ' + ', '.join([x and x or "''(empty name)''" for x in notimportedfields]) + '.\n' | |
| 200 | - | |
| 201 | - def start_process_row(self, row_idx, ticket_id): | |
| 202 | - from ticket import PatchedTicket | |
| 203 | - self.ticket = None | |
| 204 | - self.cells = [] | |
| 205 | - self.modified = False | |
| 206 | - if ticket_id > 0: | |
| 207 | - # existing ticket. Load the ticket, to see which fields will be modified | |
| 208 | - self.ticket = PatchedTicket(self.env, ticket_id) | |
| 209 | - | |
| 210 | - | |
| 211 | - def process_cell(self, column, cell): | |
| 212 | - if self.ticket and not (self.ticket.values.has_key(column) and self.ticket[column] == cell): | |
| 213 | - self.cells.append( { 'col': column, 'value': cell, 'style': 'modified-' + column }) | |
| 214 | - self.modified = True | |
| 215 | - else: | |
| 216 | - self.cells.append( { 'col': column, 'value': cell, 'style': column }) | |
| 217 | - | |
| 218 | - def end_process_row(self): | |
| 219 | - odd = len(self.data['rows']) % 2 | |
| 220 | - if self.ticket: | |
| 221 | - if self.modified: | |
| 222 | - self.modifiedcount += 1 | |
| 223 | - style = '' | |
| 224 | - ticket = self.ticket.id | |
| 225 | - else: | |
| 226 | - self.notmodifiedcount += 1 | |
| 227 | - style = '' | |
| 228 | - ticket = self.ticket.id | |
| 229 | - else: | |
| 230 | - self.added += 1 | |
| 231 | - style = odd and 'color-new-odd' or 'color-new-even' | |
| 232 | - ticket = '(new)' | |
| 233 | - | |
| 234 | - self.data['rows'].append({ 'style': style, 'cells': [{ 'col': 'ticket', 'value': ticket, 'style': '' }] + self.cells }) | |
| 235 | - | |
| 236 | - | |
| 237 | - def process_new_lookups(self, newvalues): | |
| 238 | - self.message += ' * Some lookup values are not found and will be added to the possible list of values:\n\n' | |
| 239 | - self.message += " ||'''field'''||'''New values'''||\n" | |
| 240 | - for field, values in newvalues.iteritems(): | |
| 241 | - if values == []: | |
| 242 | - continue | |
| 243 | - self.message += " ||" + field.capitalize() + "||" + ', '.join(values) + "||\n" | |
| 244 | - | |
| 245 | - | |
| 246 | - def process_new_users(self, newusers): | |
| 247 | - self.message += ' * Some user names do not exist in the system: ' + ', '.join(newusers) + '. Make sure that they are valid users.\n' | |
| 248 | - | |
| 249 | - def end_process(self, numrows): | |
| 250 | - self.message = 'Scroll to see a preview of the tickets as they will be imported. If the data is correct, select the \'\'\'Execute Import\'\'\' button.\n' + ' * ' + str(numrows) + ' tickets will be imported (' + str(self.added) + ' added, ' + str(self.modifiedcount) + ' modified, ' + str(self.notmodifiedcount) + ' unchanged).\n' + self.message | |
| 251 | - self.data['message'] = Markup(self.styles) + wiki_to_html(self.message, self.env, self.req) + Markup('<br/>') | |
| 252 | - | |
| 253 | - return 'import_preview.html', self.data, None | |
| 254 | - | |
| 1 | +# -*- coding: utf-8 -*- | |
| 2 | +# | |
| 3 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 4 | +# Author: Francois Granade - fg at nexb dot com | |
| 5 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 6 | +# | |
| 7 | + | |
| 8 | +import time | |
| 9 | + | |
| 10 | +from trac.ticket import Ticket, model | |
| 11 | +from trac.util import get_reporter_id | |
| 12 | +from trac.util.datefmt import format_datetime | |
| 13 | +from trac.util.html import Markup | |
| 14 | +from trac.wiki import wiki_to_html | |
| 15 | + | |
| 16 | + | |
| 17 | +class ImportProcessor(object): | |
| 18 | + def __init__(self, env, req, filename, tickettime): | |
| 19 | + self.parent_tid = 0 | |
| 20 | + self.env = env | |
| 21 | + self.req = req | |
| 22 | + self.filename = filename | |
| 23 | + self.modifiedcount = 0 | |
| 24 | + self.notmodifiedcount = 0 | |
| 25 | + self.added = 0 | |
| 26 | + | |
| 27 | + # TODO: check that the tickets haven't changed since preview | |
| 28 | + self.tickettime = tickettime | |
| 29 | + | |
| 30 | + # Keep the db to commit it all at once at the end | |
| 31 | + self.db = self.env.get_db_cnx() | |
| 32 | + self.missingemptyfields = None | |
| 33 | + self.missingdefaultedfields = None | |
| 34 | + self.computedfields = None | |
| 35 | + | |
| 36 | + def start(self, importedfields, reconciliate_by_owner_also, has_comments): | |
| 37 | + self.parent_tid = 0 | |
| 38 | + pass | |
| 39 | + | |
| 40 | + def process_missing_fields(self, missingfields, missingemptyfields, missingdefaultedfields, computedfields): | |
| 41 | + self.missingemptyfields = missingemptyfields | |
| 42 | + self.missingdefaultedfields = missingdefaultedfields | |
| 43 | + self.computedfields = computedfields | |
| 44 | + | |
| 45 | + def process_notimported_fields(self, notimportedfields): | |
| 46 | + pass | |
| 47 | + | |
| 48 | + def process_comment_field(self, comment): | |
| 49 | + pass | |
| 50 | + | |
| 51 | + def start_process_row(self, row_idx, ticket_id): | |
| 52 | + from ticket import PatchedTicket | |
| 53 | + if ticket_id > 0: | |
| 54 | + # existing ticket | |
| 55 | + self.ticket = PatchedTicket(self.env, tkt_id=ticket_id, db=self.db) | |
| 56 | + | |
| 57 | + # 'Ticket.time_changed' is a datetime in 0.11, and an int in 0.10. | |
| 58 | + # if we have trac.util.datefmt.to_datetime, we're likely with 0.11 | |
| 59 | + try: | |
| 60 | + from trac.util.datefmt import to_timestamp | |
| 61 | + time_changed = to_timestamp(self.ticket.time_changed) | |
| 62 | + except ImportError: | |
| 63 | + time_changed = int(self.ticket.time_changed) | |
| 64 | + | |
| 65 | + if time_changed > self.tickettime: | |
| 66 | + # just in case, verify if it wouldn't be a ticket that has been modified in the future | |
| 67 | + # (of course, it shouldn't happen... but who know). If it's the case, don't report it as an error | |
| 68 | + if time_changed < int(time.time()): | |
| 69 | + # TODO: this is not working yet... | |
| 70 | + # | |
| 71 | + #raise TracError("Sorry, can not execute the import. " | |
| 72 | + #"The ticket #" + str(ticket_id) + " has been modified by someone else " | |
| 73 | + #"since preview. You must re-upload and preview your file to avoid overwriting the other changes.") | |
| 74 | + pass | |
| 75 | + | |
| 76 | + else: | |
| 77 | + self.ticket = PatchedTicket(self.env, db=self.db) | |
| 78 | + self.comment = '' | |
| 79 | + | |
| 80 | + def process_cell(self, column, cell): | |
| 81 | + cell = unicode(cell) | |
| 82 | + # this will ensure that the changes are logged, see model.py Ticket.__setitem__ | |
| 83 | + self.ticket[column.lower()] = cell | |
| 84 | + | |
| 85 | + def process_comment(self, comment): | |
| 86 | + self.comment = comment | |
| 87 | + | |
| 88 | + def end_process_row(self, indent): | |
| 89 | + try: | |
| 90 | + # 'when' is a datetime in 0.11, and an int in 0.10. | |
| 91 | + # if we have trac.util.datefmt.to_datetime, we're likely with 0.11 | |
| 92 | + from trac.util.datefmt import to_datetime | |
| 93 | + tickettime = to_datetime(self.tickettime) | |
| 94 | + except ImportError: | |
| 95 | + tickettime = self.tickettime | |
| 96 | + | |
| 97 | + if self.ticket.id == None: | |
| 98 | + for f in self.missingemptyfields: | |
| 99 | + if self.ticket.values.has_key(f) and self.ticket[f] == None: | |
| 100 | + self.ticket[f] = '' | |
| 101 | + | |
| 102 | + if self.comment: | |
| 103 | + self.ticket['description'] = self.ticket['description'] + "\n[[BR]][[BR]]\n''Batch insert from file " + self.filename + ":''\n" + self.comment | |
| 104 | + | |
| 105 | + for f in self.computedfields: | |
| 106 | + if self.computedfields[f] != None and self.computedfields[f]['set']: | |
| 107 | + self.ticket[f] = self.computedfields[f]['value'] | |
| 108 | + | |
| 109 | + | |
| 110 | + self.added += 1 | |
| 111 | + | |
| 112 | + if (indent!=0) and (self.parent_tid!=0) and ('parents' in self.env.config['ticket-custom']): | |
| 113 | + self.ticket['parents'] = str(self.parent_tid) | |
| 114 | + | |
| 115 | + self.ticket.insert(when=tickettime, db=self.db) | |
| 116 | + | |
| 117 | + if indent==0: | |
| 118 | + self.parent_tid = self.ticket.id | |
| 119 | + | |
| 120 | + | |
| 121 | + else: | |
| 122 | + if self.comment: | |
| 123 | + message = "''Batch update from file " + self.filename + ":'' " + self.comment | |
| 124 | + else: | |
| 125 | + message = "''Batch update from file " + self.filename + "''" | |
| 126 | + if self.ticket.is_modified() or self.comment: | |
| 127 | + self.modifiedcount += 1 | |
| 128 | + self.ticket.save_changes(get_reporter_id(self.req), message, when=tickettime, db=self.db) # TODO: handle cnum, cnum = ticket.values['cnum'] + 1) | |
| 129 | + else: | |
| 130 | + self.notmodifiedcount += 1 | |
| 131 | + | |
| 132 | + self.ticket = None | |
| 133 | + | |
| 134 | + def process_new_lookups(self, newvalues): | |
| 135 | + for field, names in newvalues.iteritems(): | |
| 136 | + if names == []: | |
| 137 | + continue | |
| 138 | + if field == 'component': | |
| 139 | + class CurrentLookupEnum(model.Component): | |
| 140 | + pass | |
| 141 | + elif field == 'milestone': | |
| 142 | + class CurrentLookupEnum(model.Milestone): | |
| 143 | + pass | |
| 144 | + elif field == 'version': | |
| 145 | + class CurrentLookupEnum(model.Version): | |
| 146 | + pass | |
| 147 | + else: | |
| 148 | + class CurrentLookupEnum(model.AbstractEnum): | |
| 149 | + # here, you shouldn't put 'self.' before the class field. | |
| 150 | + type = field | |
| 151 | + | |
| 152 | + for name in names: | |
| 153 | + lookup = CurrentLookupEnum(self.env, db=self.db) | |
| 154 | + lookup.name = name | |
| 155 | + lookup.insert() | |
| 156 | + | |
| 157 | + def process_new_users(self, newusers): | |
| 158 | + pass | |
| 159 | + | |
| 160 | + def end_process(self, numrows): | |
| 161 | + self.db.commit() | |
| 162 | + | |
| 163 | + data = {} | |
| 164 | + data['title'] = 'インポート終了' | |
| 165 | + #data['report.title'] = data['title'].lower() | |
| 166 | + | |
| 167 | + message = 'Successfully imported ' + str(numrows) + ' tickets (' + str(self.added) + ' added, ' + str(self.modifiedcount) + ' modified, ' + str(self.notmodifiedcount) + ' unchanged).' | |
| 168 | + | |
| 169 | + data['message'] = Markup("<style type=\"text/css\">#report-notfound { display:none; }</style>\n") + wiki_to_html(message, self.env, self.req) | |
| 170 | + | |
| 171 | + return 'import_preview.html', data, None | |
| 172 | + | |
| 173 | + | |
| 174 | +class PreviewProcessor(object): | |
| 175 | + | |
| 176 | + def __init__(self, env, req): | |
| 177 | + self.env = env | |
| 178 | + self.req = req | |
| 179 | + self.data = {'rows': []} | |
| 180 | + self.ticket = None | |
| 181 | + self.modified = False | |
| 182 | + self.styles = '' | |
| 183 | + self.duplicatessumaries = [] | |
| 184 | + self.modifiedcount = 0 | |
| 185 | + self.notmodifiedcount = 0 | |
| 186 | + self.added = 0 | |
| 187 | + | |
| 188 | + def start(self, importedfields, reconciliate_by_owner_also, has_comments): | |
| 189 | + self.data['title'] = 'インポートのプレビュー' | |
| 190 | + | |
| 191 | + self.message = '' | |
| 192 | + | |
| 193 | + if 'ticket' in [f.lower() for f in importedfields]: | |
| 194 | + self.message += ' * A \'\'\'ticket\'\'\' column was found: Existing tickets will be updated with the values from the file. Values that are changing appear in italics in the preview below.\n' | |
| 195 | + elif 'id' in [f.lower() for f in importedfields]: | |
| 196 | + self.message += ' * A \'\'\'id\'\'\' column was found: Existing tickets will be updated with the values from the file. Values that are changing appear in italics in the preview below.\n' | |
| 197 | + else: | |
| 198 | + self.message += ' * A \'\'\'ticket\'\'\' column was not found: tickets will be reconciliated by summary' + (reconciliate_by_owner_also and ' and by owner' or '') + '. If an existing ticket with the same summary' + (reconciliate_by_owner_also and ' and the same owner' or '') + ' is found, values that are changing appear in italics in the preview below. If no ticket with same summary ' + (reconciliate_by_owner_also and ' and same owner' or '') + 'is found, the whole line appears in italics below, and a new ticket will be added.\n' | |
| 199 | + | |
| 200 | + self.data['headers'] = [{ 'col': 'ticket', 'title': 'ticket' }] | |
| 201 | + # we use one more color to set a style for all fields in a row... the CS templates happens 'color' + color + '-odd' | |
| 202 | + self.styles = "<style type=\"text/css\">\n.ticket-imported, .modified-ticket-imported { width: 40px; }\n" | |
| 203 | + self.styles += ".color-new-odd td, .color-new-even td, .modified-ticket-imported" | |
| 204 | + columns = importedfields[:] | |
| 205 | + if has_comments: | |
| 206 | + columns.append('comment') | |
| 207 | + | |
| 208 | + for col in columns: | |
| 209 | + if col.lower() != 'ticket' and col.lower() != 'id': | |
| 210 | + title=col.capitalize() | |
| 211 | + self.data['headers'].append({ 'col': col, 'title': title }) | |
| 212 | + self.styles += ", .modified-%s" % col | |
| 213 | + self.styles += " { font-style: italic; }\n" | |
| 214 | + self.styles += "</style>\n" | |
| 215 | + | |
| 216 | + # This could be simplified... | |
| 217 | + def process_missing_fields(self, missingfields, missingemptyfields, missingdefaultedfields, computedfields): | |
| 218 | + self.message += ' * Some Trac fields are not present in the import. They will default to:\n\n' | |
| 219 | + self.message += " ||'''field'''||'''Default value'''||\n" | |
| 220 | + if missingemptyfields != []: | |
| 221 | + self.message += ' ||' + ', '.join([x.capitalize() for x in missingemptyfields]) + '||' + "''(Empty value)''" + '||\n' | |
| 222 | + | |
| 223 | + if missingdefaultedfields != []: | |
| 224 | + for f in missingdefaultedfields: | |
| 225 | + self.message += ' ||' + f.capitalize() + '||' + str(computedfields[f]['value']) + '||\n' | |
| 226 | + | |
| 227 | + self.message += '(You can change some of these default values in the Trac Admin module, if you are administrator; or you can add the corresponding column to your spreadsheet and re-upload it).\n' | |
| 228 | + | |
| 229 | + def process_notimported_fields(self, notimportedfields): | |
| 230 | + self.message += ' * Some fields will not be imported because they don\'t exist in Trac: ' + ', '.join([x and x or "''(empty name)''" for x in notimportedfields]) + '.\n' | |
| 231 | + | |
| 232 | + def process_comment_field(self, comment): | |
| 233 | + self.message += ' * The field "%s" will be used as comment when modifying tickets, and appended to the description for new tickets.\n' % comment | |
| 234 | + | |
| 235 | + def start_process_row(self, row_idx, ticket_id): | |
| 236 | + from ticket import PatchedTicket | |
| 237 | + self.ticket = None | |
| 238 | + self.cells = [] | |
| 239 | + self.modified = False | |
| 240 | + if ticket_id > 0: | |
| 241 | + # existing ticket. Load the ticket, to see which fields will be modified | |
| 242 | + self.ticket = PatchedTicket(self.env, ticket_id) | |
| 243 | + | |
| 244 | + | |
| 245 | + def process_cell(self, column, cell): | |
| 246 | + if self.ticket and not (self.ticket.values.has_key(column) and self.ticket[column] == cell): | |
| 247 | + self.cells.append( { 'col': column, 'value': cell, 'style': 'modified-' + column }) | |
| 248 | + self.modified = True | |
| 249 | + else: | |
| 250 | + self.cells.append( { 'col': column, 'value': cell, 'style': column }) | |
| 251 | + | |
| 252 | + def process_comment(self, comment): | |
| 253 | + column = 'comment' | |
| 254 | + self.cells.append( { 'col': column, 'value': comment, 'style': column }) | |
| 255 | + | |
| 256 | + def end_process_row(self, indent): | |
| 257 | + odd = len(self.data['rows']) % 2 | |
| 258 | + if self.ticket: | |
| 259 | + if self.modified: | |
| 260 | + self.modifiedcount += 1 | |
| 261 | + style = '' | |
| 262 | + ticket = self.ticket.id | |
| 263 | + else: | |
| 264 | + self.notmodifiedcount += 1 | |
| 265 | + style = '' | |
| 266 | + ticket = self.ticket.id | |
| 267 | + else: | |
| 268 | + self.added += 1 | |
| 269 | + style = odd and 'color-new-odd' or 'color-new-even' | |
| 270 | + ticket = '(new)' | |
| 271 | + | |
| 272 | + self.data['rows'].append({ 'style': style, 'cells': [{ 'col': 'ticket', 'value': ticket, 'style': '' }] + self.cells }) | |
| 273 | + | |
| 274 | + | |
| 275 | + def process_new_lookups(self, newvalues): | |
| 276 | + self.message += ' * Some lookup values are not found and will be added to the possible list of values:\n\n' | |
| 277 | + self.message += " ||'''field'''||'''New values'''||\n" | |
| 278 | + for field, values in newvalues.iteritems(): | |
| 279 | + if values == []: | |
| 280 | + continue | |
| 281 | + self.message += " ||" + field.capitalize() + "||" + ', '.join(values) + "||\n" | |
| 282 | + | |
| 283 | + | |
| 284 | + def process_new_users(self, newusers): | |
| 285 | + self.message += ' * Some user names do not exist in the system: ' + ', '.join(newusers) + '. Make sure that they are valid users.\n' | |
| 286 | + | |
| 287 | + def end_process(self, numrows): | |
| 288 | + self.message = 'Scroll to see a preview of the tickets as they will be imported. If the data is correct, select the \'\'\'Execute Import\'\'\' button.\n' + ' * ' + str(numrows) + ' tickets will be imported (' + str(self.added) + ' added, ' + str(self.modifiedcount) + ' modified, ' + str(self.notmodifiedcount) + ' unchanged).\n' + self.message | |
| 289 | + self.data['message'] = Markup(self.styles) + wiki_to_html(self.message, self.env, self.req) + Markup('<br/>') | |
| 290 | + | |
| 291 | + return 'import_preview.html', self.data, None | |
| 292 | + |
| @@ -1,37 +1,37 @@ | ||
| 1 | -# | |
| 2 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | -# Author: Francois Granade - fg at nexb dot com | |
| 4 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | -# | |
| 6 | - | |
| 7 | -from trac.ticket import Ticket, model | |
| 8 | - | |
| 9 | -class PatchedTicket(Ticket): | |
| 10 | - ''' patched version of the Ticket class, that doesn't make the difference between a field defaulting to an empty string, and a field not defaulted | |
| 11 | - ''' | |
| 12 | - | |
| 13 | - # TODO: report it as a bug, and/or check if it is fixed in more recent versions | |
| 14 | - def _init_defaults(self, db=None): | |
| 15 | - for field in self.fields: | |
| 16 | - default = None | |
| 17 | - if field['name'] in ['resolution']: | |
| 18 | - # Ignore for new - only change through workflow | |
| 19 | - pass | |
| 20 | - elif not field.get('custom'): | |
| 21 | - default = self.env.config.get('ticket', | |
| 22 | - 'default_' + field['name'], None) | |
| 23 | - else: | |
| 24 | - default = field.get('value') | |
| 25 | - options = field.get('options') | |
| 26 | - if default and default != '' and options and default not in options: | |
| 27 | - try: | |
| 28 | - default = options[int(default)] | |
| 29 | - except (ValueError, IndexError): | |
| 30 | - self.env.log.warning('Invalid default value "%s" ' | |
| 31 | - 'for custom field "%s"' | |
| 32 | - % (default, field['name'])) | |
| 33 | - if default or default == '': | |
| 34 | - self.values.setdefault(field['name'], default) | |
| 35 | - | |
| 36 | - def is_modified(self): | |
| 37 | - return self._old | |
| 1 | +# | |
| 2 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | +# Author: Francois Granade - fg at nexb dot com | |
| 4 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | +# | |
| 6 | + | |
| 7 | +from trac.ticket import Ticket, model | |
| 8 | + | |
| 9 | +class PatchedTicket(Ticket): | |
| 10 | + ''' patched version of the Ticket class, that doesn't make the difference between a field defaulting to an empty string, and a field not defaulted | |
| 11 | + ''' | |
| 12 | + | |
| 13 | + # TODO: report it as a bug, and/or check if it is fixed in more recent versions | |
| 14 | + def _init_defaults(self, db=None): | |
| 15 | + for field in self.fields: | |
| 16 | + default = None | |
| 17 | + if field['name'] in ['resolution']: | |
| 18 | + # Ignore for new - only change through workflow | |
| 19 | + pass | |
| 20 | + elif not field.get('custom'): | |
| 21 | + default = self.env.config.get('ticket', | |
| 22 | + 'default_' + field['name'], None) | |
| 23 | + else: | |
| 24 | + default = field.get('value') | |
| 25 | + options = field.get('options') | |
| 26 | + if default and default != '' and options and default not in options: | |
| 27 | + try: | |
| 28 | + default = options[int(default)] | |
| 29 | + except (ValueError, IndexError): | |
| 30 | + self.env.log.warning('Invalid default value "%s" ' | |
| 31 | + 'for custom field "%s"' | |
| 32 | + % (default, field['name'])) | |
| 33 | + if default or default == '': | |
| 34 | + self.values.setdefault(field['name'], default) | |
| 35 | + | |
| 36 | + def is_modified(self): | |
| 37 | + return self._old |
| @@ -1,82 +1,82 @@ | ||
| 1 | -# | |
| 2 | -# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | -# Author: Francois Granade - fg at nexb dot com | |
| 4 | -# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | -# | |
| 6 | - | |
| 7 | -import csv | |
| 8 | - | |
| 9 | -from trac.core import TracError | |
| 10 | - | |
| 11 | - | |
| 12 | -def get_reader(filename, sheet_index = 1): | |
| 13 | - # NOTE THAT the sheet index is 1-based ! | |
| 14 | - # KISS - keep it simple: if it can be opened as XLS, do, otherwise try as CSV. | |
| 15 | - try: | |
| 16 | - return XLSReader(filename, sheet_index) | |
| 17 | - except ImportError: | |
| 18 | - try: | |
| 19 | - return CSVReader(filename) | |
| 20 | - except: | |
| 21 | - raise TracError('XLS reading is not configured, and this file is not a valid CSV file: unable to read file.') | |
| 22 | - except IndexError: | |
| 23 | - raise TracError('The sheet index (' + str(sheet_index) + ') does not seem to correspond to an existing sheet in the spreadsheet') | |
| 24 | - except: | |
| 25 | - try: | |
| 26 | - return CSVReader(filename) | |
| 27 | - except: | |
| 28 | - raise TracError('Unable to read this file, does not seem to be a valid Excel or CSV file.') | |
| 29 | - | |
| 30 | - | |
| 31 | -class CP1252DictReader(csv.DictReader): | |
| 32 | - def next(self): | |
| 33 | - d = csv.DictReader.next(self) | |
| 34 | - return dict([(key, value.decode('cp1252', 'replace')) for key, value in d.iteritems()]) | |
| 35 | - | |
| 36 | -class CSVReader(object): | |
| 37 | - def __init__(self, filename): | |
| 38 | - self.file = open(filename, "rb") | |
| 39 | - reader = csv.reader(self.file) | |
| 40 | - self.csvfields = reader.next() | |
| 41 | - | |
| 42 | - def get_sheet_count(): | |
| 43 | - return 1 | |
| 44 | - | |
| 45 | - def readers(self): | |
| 46 | - return self.csvfields, CP1252DictReader(self.file, self.csvfields) | |
| 47 | - | |
| 48 | - def close(self): | |
| 49 | - self.file.close() | |
| 50 | - | |
| 51 | -class XLSReader(object): | |
| 52 | - def __init__(self, filename, sheet_index = 1): | |
| 53 | - import xlrd | |
| 54 | - book = xlrd.open_workbook(filename) | |
| 55 | - self.sheetcount = book.nsheets | |
| 56 | - self.sh = book.sheet_by_index(sheet_index - 1) | |
| 57 | - | |
| 58 | - def get_sheet_count(): | |
| 59 | - return self.sheetcount | |
| 60 | - | |
| 61 | - def readers(self): | |
| 62 | - # TODO: do something with sh.name. Probably add it as a column. | |
| 63 | - # TODO: read the other sheets. What if they don't have the same columns ? | |
| 64 | - header = [] | |
| 65 | - for cx in range(self.sh.ncols): | |
| 66 | - header.append(self.sh.cell_value(rowx=0, colx=cx)) | |
| 67 | - | |
| 68 | - data = [] | |
| 69 | - for rx in range(self.sh.nrows): | |
| 70 | - if rx == 0: | |
| 71 | - continue | |
| 72 | - row = {} | |
| 73 | - i = 0 | |
| 74 | - for cx in range(self.sh.ncols): | |
| 75 | - row[header[i]] = self.sh.cell_value(rowx=rx, colx=cx) | |
| 76 | - i += 1 | |
| 77 | - data.append(row) | |
| 78 | - | |
| 79 | - return header, data | |
| 80 | - | |
| 81 | - def close(self): | |
| 82 | - pass | |
| 1 | +# | |
| 2 | +# Copyright (c) 2007-2008 by nexB, Inc. http://www.nexb.com/ - All rights reserved. | |
| 3 | +# Author: Francois Granade - fg at nexb dot com | |
| 4 | +# Licensed under the same license as Trac - http://trac.edgewall.org/wiki/TracLicense | |
| 5 | +# | |
| 6 | + | |
| 7 | +import csv | |
| 8 | + | |
| 9 | +from trac.core import TracError | |
| 10 | + | |
| 11 | + | |
| 12 | +def get_reader(filename, sheet_index = 1): | |
| 13 | + # NOTE THAT the sheet index is 1-based ! | |
| 14 | + # KISS - keep it simple: if it can be opened as XLS, do, otherwise try as CSV. | |
| 15 | + try: | |
| 16 | + return XLSReader(filename, sheet_index) | |
| 17 | + except ImportError: | |
| 18 | + try: | |
| 19 | + return CSVReader(filename) | |
| 20 | + except: | |
| 21 | + raise TracError('XLS reading is not configured, and this file is not a valid CSV file: unable to read file.') | |
| 22 | + except IndexError: | |
| 23 | + raise TracError('The sheet index (' + str(sheet_index) + ') does not seem to correspond to an existing sheet in the spreadsheet') | |
| 24 | + except: | |
| 25 | + try: | |
| 26 | + return CSVReader(filename) | |
| 27 | + except: | |
| 28 | + raise TracError('Unable to read this file, does not seem to be a valid Excel or CSV file.') | |
| 29 | + | |
| 30 | + | |
| 31 | +class CP1252DictReader(csv.DictReader): | |
| 32 | + def next(self): | |
| 33 | + d = csv.DictReader.next(self) | |
| 34 | + return dict([(key, value.decode('cp1252', 'replace')) for key, value in d.iteritems()]) | |
| 35 | + | |
| 36 | +class CSVReader(object): | |
| 37 | + def __init__(self, filename): | |
| 38 | + self.file = open(filename, "rb") | |
| 39 | + reader = csv.reader(self.file) | |
| 40 | + self.csvfields = reader.next() | |
| 41 | + | |
| 42 | + def get_sheet_count(): | |
| 43 | + return 1 | |
| 44 | + | |
| 45 | + def readers(self): | |
| 46 | + return self.csvfields, CP1252DictReader(self.file, self.csvfields) | |
| 47 | + | |
| 48 | + def close(self): | |
| 49 | + self.file.close() | |
| 50 | + | |
| 51 | +class XLSReader(object): | |
| 52 | + def __init__(self, filename, sheet_index = 1): | |
| 53 | + import xlrd | |
| 54 | + book = xlrd.open_workbook(filename) | |
| 55 | + self.sheetcount = book.nsheets | |
| 56 | + self.sh = book.sheet_by_index(sheet_index - 1) | |
| 57 | + | |
| 58 | + def get_sheet_count(): | |
| 59 | + return self.sheetcount | |
| 60 | + | |
| 61 | + def readers(self): | |
| 62 | + # TODO: do something with sh.name. Probably add it as a column. | |
| 63 | + # TODO: read the other sheets. What if they don't have the same columns ? | |
| 64 | + header = [] | |
| 65 | + for cx in range(self.sh.ncols): | |
| 66 | + header.append(self.sh.cell_value(rowx=0, colx=cx)) | |
| 67 | + | |
| 68 | + data = [] | |
| 69 | + for rx in range(self.sh.nrows): | |
| 70 | + if rx == 0: | |
| 71 | + continue | |
| 72 | + row = {} | |
| 73 | + i = 0 | |
| 74 | + for cx in range(self.sh.ncols): | |
| 75 | + row[header[i]] = self.sh.cell_value(rowx=rx, colx=cx) | |
| 76 | + i += 1 | |
| 77 | + data.append(row) | |
| 78 | + | |
| 79 | + return header, data | |
| 80 | + | |
| 81 | + def close(self): | |
| 82 | + pass |
Fork