Package etl :: Module transformer
[hide private]
[frames] | no frames]

Source Code for Module etl.transformer

  1  # -*- encoding: utf-8 -*- 
  2  ############################################################################## 
  3  # 
  4  #    ETL system- Extract Transfer Load system 
  5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
  6  #    $Id$ 
  7  # 
  8  #    This program is free software: you can redistribute it and/or modify 
  9  #    it under the terms of the GNU General Public License as published by 
 10  #    the Free Software Foundation, either version 3 of the License, or 
 11  #    (at your option) any later version. 
 12  # 
 13  #    This program is distributed in the hope that it will be useful, 
 14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  #    GNU General Public License for more details. 
 17  # 
 18  #    You should have received a copy of the GNU General Public License 
 19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 20  # 
 21  ############################################################################## 
 22  """  
 23   ETL Process. 
 24    
 25   The class provides transformeration process. 
 26    
 27   Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). 
 28   GNU General Public License. 
 29   
 30  """ 
 31  import datetime 
 32  import logger 
 33   
 34  DATE_FORMAT = '%Y-%m-%d' 
 35  TIME_FORMAT = '%H:%M:%S' 
 36  DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'     
 37   
38 -class transformer(object):
39 """ 40 Transfer data into different type. 41 Pass description like :- 42 - INTEGER : convert string to Integer object. 43 - FLOAT : convert string to float object. 44 - LONG : convert string to long integer object. 45 - COMPLEX : convert string to complex number. 46 - STRING : convert string to string. 47 - DATE : convert string to datetime.date object. 48 - DATETIME : convert string to datetime.datetime object. 49 - TIME : convert string to datetime.time object. 50 - BOOLEAN : convert string to boolean object. 51 Example :- 52 datas = [{'id': '1', 'name': 'abc', 'invoice_date': '2009-10-20', 'invoice_amount': '200.00', 'is_paid': '1'}] 53 description= {'id': etl.transformer.LONG, 'name': etl.transformer.STRING, 'invoice_date': etl.transformer.DATE, 'invoice_amount': etl.transformer.FLOAT, 'is_paid': etl.transformer.BOOLEAN} 54 return = [{'id': 1, 'name': 'abc', 'invoice_date': datetime.date object (2009, 10, 20), 'invoice_amount': 200.00, 'is_paid': True}] 55 """ 56 INTEGER = 'int' 57 STRING = 'str' 58 DATE = 'date' 59 DATETIME = 'datetime' 60 TIME = 'time' 61 FLOAT = 'float' 62 LONG = 'long' 63 COMPLEX = 'complex' 64 BOOLEAN = 'bool' 65 66 _transform_method = { 67 'int': int, 68 'str': unicode, 69 'date': lambda x: datetime.datetime.strptime(x, DATE_FORMAT).date(), 70 'time': lambda x: datetime.datetime.strptime(x, TIME_FORMAT).time(), 71 'datetime':lambda x: datetime.datetime.strptime(x, DATETIME_FORMAT), 72 'float': float, 73 'long': long, 74 'complex': complex, 75 'bool': bool 76 } 77
78 - def __init__(self, description):
79 self.description = description 80 self.logger = logger.logger()
81
82 - def action_error(self, e):
83 print e 84 self.logger.notifyChannel("transformer", logger.LOG_ERROR, 85 str(self) + ' : ' + str(e)) 86 return None
87
88 - def transform(self, data):
89 # TODO : TO check : data and description should have same keys. 90 try: 91 for column in data: 92 if column in self.description: 93 transform_method = self.description[column] and self._transform_method[self.description[column]] 94 data[column] = data[column] and transform_method(data[column]) or data[column] 95 return data 96 except UnicodeEncodeError, e: 97 return self.action_error(e) 98 except UnicodeDecodeError, e: 99 return self.action_error(e) 100 except TypeError, e: 101 return self.action_error(e) 102 except ValueError, e: 103 return self.action_error(e)
104