Home | Trees | Indices | Help |
---|
|
1 # -*- encoding: utf-8 -*- 2 ############################################################################## 3 # 4 # ETL system- Extract Transfer Load system 5 # Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 6 # $Id$ 7 # 8 # This program is free software: you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation, either version 3 of the License, or 11 # (at your option) any later version. 12 # 13 # This program is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with this program. If not, see <http://www.gnu.org/licenses/>. 20 # 21 ############################################################################## 22 """ 23 ETL Process. 24 25 The class provides transformeration process. 26 27 Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). 28 GNU General Public License. 29 30 """ 31 import datetime 32 import logger 33 34 DATE_FORMAT = '%Y-%m-%d' 35 TIME_FORMAT = '%H:%M:%S' 36 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' 3739 """ 40 Transfer data into different type. 41 Pass description like :- 42 - INTEGER : convert string to Integer object. 43 - FLOAT : convert string to float object. 44 - LONG : convert string to long integer object. 45 - COMPLEX : convert string to complex number. 46 - STRING : convert string to string. 47 - DATE : convert string to datetime.date object. 48 - DATETIME : convert string to datetime.datetime object. 49 - TIME : convert string to datetime.time object. 50 - BOOLEAN : convert string to boolean object. 51 Example :- 52 datas = [{'id': '1', 'name': 'abc', 'invoice_date': '2009-10-20', 'invoice_amount': '200.00', 'is_paid': '1'}] 53 description= {'id': etl.transformer.LONG, 'name': etl.transformer.STRING, 'invoice_date': etl.transformer.DATE, 'invoice_amount': etl.transformer.FLOAT, 'is_paid': etl.transformer.BOOLEAN} 54 return = [{'id': 1, 'name': 'abc', 'invoice_date': datetime.date object (2009, 10, 20), 'invoice_amount': 200.00, 'is_paid': True}] 55 """ 56 INTEGER = 'int' 57 STRING = 'str' 58 DATE = 'date' 59 DATETIME = 'datetime' 60 TIME = 'time' 61 FLOAT = 'float' 62 LONG = 'long' 63 COMPLEX = 'complex' 64 BOOLEAN = 'bool' 65 66 _transform_method = { 67 'int': int, 68 'str': unicode, 69 'date': lambda x: datetime.datetime.strptime(x, DATE_FORMAT).date(), 70 'time': lambda x: datetime.datetime.strptime(x, TIME_FORMAT).time(), 71 'datetime':lambda x: datetime.datetime.strptime(x, DATETIME_FORMAT), 72 'float': float, 73 'long': long, 74 'complex': complex, 75 'bool': bool 76 } 77 8110483 print e 84 self.logger.notifyChannel("transformer", logger.LOG_ERROR, 85 str(self) + ' : ' + str(e)) 86 return None87 89 # TODO : TO check : data and description should have same keys. 90 try: 91 for column in data: 92 if column in self.description: 93 transform_method = self.description[column] and self._transform_method[self.description[column]] 94 data[column] = data[column] and transform_method(data[column]) or data[column] 95 return data 96 except UnicodeEncodeError, e: 97 return self.action_error(e) 98 except UnicodeDecodeError, e: 99 return self.action_error(e) 100 except TypeError, e: 101 return self.action_error(e) 102 except ValueError, e: 103 return self.action_error(e)
Home | Trees | Indices | Help |
---|
Generated by Epydoc 3.0beta1 on Wed Apr 29 20:37:24 2009 | http://epydoc.sourceforge.net |