Package etl :: Package component :: Package transform :: Module data_filter'
[hide private]
[frames] | no frames]

Source Code for Module etl.component.transform.data_filter'

 1  # -*- encoding: utf-8 -*- 
 2  ############################################################################## 
 3  # 
 4  #    ETL system- Extract Transfer Load system 
 5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
 6  #    $Id$ 
 7  # 
 8  #    This program is free software: you can redistribute it and/or modify 
 9  #    it under the terms of the GNU General Public License as published by 
10  #    the Free Software Foundation, either version 3 of the License, or 
11  #    (at your option) any later version. 
12  # 
13  #    This program is distributed in the hope that it will be useful, 
14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
16  #    GNU General Public License for more details. 
17  # 
18  #    You should have received a copy of the GNU General Public License 
19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
20  # 
21  ############################################################################## 
22  """ 
23  Data filter component. 
24  """ 
25   
26  from etl.component import component 
27  import datetime 
28   
29 -class data_filter(component):
30 """ 31 Data filter component. 32 """ 33
34 - def __init__(self, filter_criteria, name='component.transfer.data_filter', transformer=None, row_limit=0):
35 36 """ 37 Required Parameters 38 filter_criteria : Dictionary of filtering criteria. 39 40 Extra Parameters 41 name : Name of component. 42 transformer : Transformer object to transform string data into particular type. 43 """ 44 super(data_filter, self).__init__(name=name, transformer=transformer,row_limit=row_limit) 45 self._type = 'component.transfer.data_filter' 46 self.filter_criteria = filter_criteria
47
48 - def __copy__(self):
49 res = data_filter(self.filter_criteria, self.name, self.transformer, self.row_limit) 50 return res
51
52 - def process(self):
53 datas = [] 54 for channel, trans in self.input_get().items(): 55 for iterator in trans: 56 for d in iterator: 57 if self.transformer: 58 d = self.transformer.transform(d) 59 filter = '' 60 for filter_data in self.filter_criteria: 61 val = d[filter_data['name']] 62 _filter = filter_data.get('filter', False) 63 if val and _filter: 64 val = eval((_filter) % d) 65 filter += " %s %s %s %s" % (repr(val), filter_data['operator'], filter_data['operand'], filter_data.get('condition', '')) 66 if self.transformer: 67 d = self.transformer.transform(d) 68 if eval(filter): 69 yield d, 'main' 70 else: 71 yield d, 'invalid'
72 -def test():
73 from etl_test import etl_test 74 from etl import transformer 75 import etl 76 openobject_partner=etl.connector.openobject_connector('http://localhost:8869', 'trunk', 'admin', 'admin',con_type='xmlrpc') 77 transformer_description= {'title':transformer.STRING,'name':transformer.STRING,'street':transformer.STRING,'street2':transformer.STRING,'birthdate':transformer.DATE} 78 transformer=transformer(transformer_description) 79 openobject_in1= etl.component.input.openobject_in( 80 openobject_partner,'res.partner.address', 81 fields=['partner_id','title', 'name', 'street', 'street2' , 'phone' , 'city' , 'zip' ,'state_id' , 'country_id' , 'mobile', 'birthdate'], 82 transformer=transformer) 83 filter_criteria=[ 84 {'name':'Partner','filter':'"%(Partner)s".lower() or ""','operator':'==','operand':"'leclerc'",'condition':'or'}, 85 {'name':'Address Name','operator':'==','operand':"'Fabien Pinckaers'"} 86 ] 87 88 test1 = data_filter_component=etl_test.etl_component_test(etl.component.transform.data_filter(filter_criteria,transformer=transformer)) 89 res = test1.output() 90 print res
91 if __name__ == '__main__': 92 test() 93