Package etl :: Package component :: Package input :: Module csv_in
[hide private]
[frames] | no frames]

Source Code for Module etl.component.input.csv_in

 1  # -*- encoding: utf-8 -*- 
 2  ############################################################################## 
 3  # 
 4  #    ETL system- Extract Transfer Load system 
 5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
 6  #    $Id$ 
 7  # 
 8  #    This program is free software: you can redistribute it and/or modify 
 9  #    it under the terms of the GNU General Public License as published by 
10  #    the Free Software Foundation, either version 3 of the License, or 
11  #    (at your option) any later version. 
12  # 
13  #    This program is distributed in the hope that it will be useful, 
14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
16  #    GNU General Public License for more details. 
17  # 
18  #    You should have received a copy of the GNU General Public License 
19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
20  # 
21  ############################################################################## 
22  """ 
23   To read data from csv file. 
24   
25   Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).  
26   GNU General Public License. 
27  """ 
28  from etl.component import component 
29   
30 -class csv_in(component):
31 """ 32 This is an ETL Component that is used to read data from csv file. Its type is data component. 33 Its computing peformance is streamline. 34 It has two flows 35 Input Flows : 0. 36 * .* : Nothing. 37 Output Flows : 0-x. 38 * .* : Returns the main flow with data from csv file. 39 """ 40
41 - def __init__(self, fileconnector, csv_params={}, name='component.input.csv_in', transformer=None, row_limit=0):
42 """ 43 Required Parameters 44 fileconnector : Localfile connector. 45 46 Extra Parameters 47 name : Name of Component. 48 transformer : Transformer object to transform string data into particular object. 49 row_limit : Limited records are sent to destination if row limit is specified. If row limit is 0, all records are sent. 50 csv_param : To specify other csv parameter like fieldnames , restkey , restval etc. 51 """ 52 super(csv_in, self).__init__(name=name, connector=fileconnector, transformer=transformer, row_limit=row_limit) 53 self._type = 'component.input.csv_in' 54 self.csv_params = csv_params
55
56 - def __copy__(self):
57 res = csv_in(self.connector , self.csv_params, self.name, self.transformer, self.row_limit) 58 return res 59
60 - def end(self):
61 super(csv_in, self).end() 62 if self.fp: 63 self.connector.close(self.fp) 64 self.fp = False
65
66 - def process(self):
67 if self.is_end(): 68 self.warning('No any Output attached') 69 import csv 70 self.fp = self.connector.open() 71 reader = csv.DictReader(self.fp, **self.csv_params) 72 for data in reader: 73 if data: 74 yield data, 'main'
75
76 -def test():
77 from etl_test import etl_test 78 import etl 79 file_conn = etl.connector.localfile('../../../demo/input/partner1.csv') 80 test = etl_test.etl_component_test(csv_in(file_conn, name='csv test')) 81 test.check_output([{'tel': '+32.81.81.37.00', 'id': '11', 'name': 'Fabien'}]) 82 res = test.output() 83 print res
84 85 if __name__ == '__main__': 86 test() 87