Package etl :: Package component :: Package output :: Module csv_out'
[hide private]
[frames] | no frames]

Source Code for Module etl.component.output.csv_out'

  1  # -*- encoding: utf-8 -*- 
  2  ############################################################################## 
  3  # 
  4  #    ETL system- Extract Transfer Load system 
  5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
  6  #    $Id$ 
  7  # 
  8  #    This program is free software: you can redistribute it and/or modify 
  9  #    it under the terms of the GNU General Public License as published by 
 10  #    the Free Software Foundation, either version 3 of the License, or 
 11  #    (at your option) any later version. 
 12  # 
 13  #    This program is distributed in the hope that it will be useful, 
 14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  #    GNU General Public License for more details. 
 17  # 
 18  #    You should have received a copy of the GNU General Public License 
 19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 20  # 
 21  ############################################################################## 
 22  """ 
 23   To write data to csv file. 
 24   
 25   Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). 
 26   GNU General Public License 
 27  """ 
 28   
 29  from etl.component import component 
 30   
 31   
32 -class csv_out(component):
33 """ 34 This is an ETL Component that writes data to csv file. 35 36 Type : Data Component. 37 Computing Performance : Streamline. 38 Input Flows : 0-x. 39 * .* : The main data flow with input data. 40 Output Flows : 0-1. 41 * main : Return all data. 42 """ 43
44 - def __init__(self, fileconnector, csv_params={}, name='component.output.csv_out', transformer=None, row_limit=0):
45 """ 46 Required Parameters 47 fileconnector : Localfile connector. 48 49 Extra Parameters 50 name : Name of Component. 51 transformer : Transformer object to transform string data into particular object. 52 row_limit : Limited records are sent to destination if row limit is specified. If row limit is 0, all records are sent. 53 csv_param : To specify other csv parameter like fieldnames , restkey , restval etc. 54 """ 55 super(csv_out, self).__init__(name=name, connector=fileconnector, transformer=transformer, row_limit=row_limit) 56 self._type = 'component.output.csv_out' 57 self.csv_params = csv_params
58
59 - def __copy__(self):
60 res = csv_out(self.connector , self.csv_params, self.name, self.transformer, self.row_limit) 61 return res
62
63 - def end(self):
64 super(csv_out, self).end() 65 if self.fp: 66 self.connector.close(self.fp) 67 self.fp = False
68
69 - def process(self):
70 if self.is_start(): 71 self.warning('No any Input attached') 72 import csv 73 datas = [] 74 self.fp = False 75 writer = False 76 for channel, trans in self.input_get().items(): 77 for iterator in trans: 78 for d in iterator: 79 if not self.fp: 80 self.fp = self.connector.open('wb+') 81 fieldnames = d.keys() 82 writer = csv.DictWriter(self.fp, fieldnames) 83 writer.writerow(dict(map(lambda x: (x, x), fieldnames))) 84 writer.writerow(d) 85 yield d, 'main'
86
87 -def test():
88 from etl_test import etl_test 89 import etl 90 file_conn = etl.connector.localfile('../../../demo/input/partner1.csv') 91 test = etl_test.etl_component_test(csv_out(file_conn, name='csv test')) 92 test.check_input({'main': [{'tel': '+32.81.81.37.00', 'id': '11', 'name': 'Fabien11'}]}) 93 test.check_output([{'tel': '+32.81.81.37.00', 'id': '11', 'name': 'Fabien11'}], 'main') 94 res = test.output() 95 print res
96 97 if __name__ == '__main__': 98 test() 99