Package etl :: Package component :: Package transform :: Module data_map
[hide private]
[frames] | no frames]

Source Code for Module etl.component.transform.data_map

  1  # -*- encoding: utf-8 -*- 
  2  ############################################################################## 
  3  # 
  4  #    ETL system- Extract Transfer Load system 
  5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
  6  #    $Id$ 
  7  # 
  8  #    This program is free software: you can redistribute it and/or modify 
  9  #    it under the terms of the GNU General Public License as published by 
 10  #    the Free Software Foundation, either version 3 of the License, or 
 11  #    (at your option) any later version. 
 12  # 
 13  #    This program is distributed in the hope that it will be useful, 
 14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  #    GNU General Public License for more details. 
 17  # 
 18  #    You should have received a copy of the GNU General Public License 
 19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 20  # 
 21  ############################################################################## 
 22  """ 
 23  Data Map component. 
 24  """ 
 25   
 26  from etl.component import component 
 27  import datetime 
 28  from etl import tools 
 29   
30 -class map(component):
31 """ 32 Data map component. 33 """ 34
35 - def __init__(self, map_criteria, preprocess=None, name='component.transfer.map', transformer=None, row_limit=0):
36 """ 37 Required Parameters 38 map_criteria : Mapping criteria. 39 40 Extra Parameters 41 name : Name of the component. 42 transformer : Transformer object to transform string data into particular object. 43 preprocess : For initializing the instance of custom variable. 44 """ 45 46 super(map, self).__init__(name=name, transformer=transformer, row_limit=row_limit) 47 self._type = 'component.transfer.map' 48 self.map_criteria = map_criteria 49 self.preprocess = preprocess
50
51 - def __copy__(self):
52 res = map(self.map_criteria, self.preprocess, self.name, self.transformer, self.row_limit) 53 return res
54
55 - def process(self):
56 channels = self.input_get() 57 datas = {} 58 if self.preprocess: 59 datas = self.preprocess(self, channels) 60 for channel, trans in channels.items(): 61 for iterator in trans: 62 for d in iterator: 63 for channel_dest, channel_value in self.map_criteria.items(): 64 result = {} 65 for key, val in channel_value.items(): 66 if val: 67 datas['main'] = d 68 datas['tools'] = tools 69 result[key] = eval(val, datas) 70 else: 71 result[key] = val 72 if self.transformer: 73 result = self.transformer.transform(result) 74 yield result, channel_dest
75
76 -def test():
77 78 from etl_test import etl_test 79 import etl 80 input_part = [ 81 {'id': 1, 'name': 'Fabien', 'country_id': 3}, 82 {'id': 2, 'name': 'Luc', 'country_id': 3}, 83 {'id': 3, 'name': 'Henry', 'country_id': 1} 84 ] 85 input_cty = [{'id': 1, 'name': 'Belgium'}, {'id': 3, 'name': 'France'}] 86 map_keys = {'main': { 87 'id': "main['id']", 88 'name': "main['id'].upper()", 89 'country': "country_var[main['country_id']]['name']" 90 }} 91 # def preprocess(self, channels): 92 # cdict = {} 93 # for trans in channels['country']: 94 # for iterator in trans: 95 # for d in iterator: 96 # cdict[d['id']] = d 97 # return {'country_var': cdict} 98 test = etl_test.etl_component_test(map(map_keys, None)) 99 test.check_input({'partner':input_part, 'countries': input_cty}) 100 test.check_output({'partner':input_part, 'countries': input_cty}) 101 print test.output()
102 103 if __name__ == '__main__': 104 test() 105