Package etl :: Package component :: Package input :: Module sql_in'
[hide private]
[frames] | no frames]

Source Code for Module etl.component.input.sql_in'

  1  # -*- encoding: utf-8 -*- 
  2  ############################################################################## 
  3  # 
  4  #    ETL system- Extract Transfer Load system 
  5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
  6  #    $Id$ 
  7  # 
  8  #    This program is free software: you can redistribute it and/or modify 
  9  #    it under the terms of the GNU General Public License as published by 
 10  #    the Free Software Foundation, either version 3 of the License, or 
 11  #    (at your option) any later version. 
 12  # 
 13  #    This program is distributed in the hope that it will be useful, 
 14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  #    GNU General Public License for more details. 
 17  # 
 18  #    You should have received a copy of the GNU General Public License 
 19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 20  # 
 21  ############################################################################## 
 22  """ 
 23   To read data from SQL database. 
 24   
 25   Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).  
 26   GNU General Public License. 
 27  """ 
 28   
 29  from etl.component import component 
 30  from etl.connector import sql_connector 
 31  import datetime 
 32   
33 -class sql_in(component):
34 """ 35 This is an ETL Component that is used to read data from SQL database. 36 37 Type : Data Component. 38 Computing Performance : Streamline. 39 Input Flows : 0. 40 * .* : Nothing. 41 Output Flows : 0-x. 42 * .* : Returns the main flow with data from csv file. 43 """ 44
45 - def __init__(self, sqlconnector, sqlquery, name='component.input.sql_in', transformer=None, row_limit=0):
46 47 """ 48 Required Parameters 49 sqlconnector : SQLconnector connector. 50 sqlquery : SQL Query 51 52 Extra Parameters 53 name : Name of Component. 54 transformer : Transformer object to transform string data into particular type. 55 row_limit : Limited records are sent to destination if row limit is specified. If row limit is 0, all records are sent. 56 """ 57 super(sql_in, self).__init__(name=name, connector=sqlconnector, transformer=transformer, row_limit=row_limit) 58 self._type = 'component.input.sql_in' 59 self.sqlquery = sqlquery
60 61
62 - def __copy__(self):
63 res = sql_in(self.connector, self.sqlquery, self.name, self.transformer, self.row_limit) 64 return res
65
66 - def end(self):
67 super(sql_in, self).end() 68 if self.sql_con: 69 self.connector.close(self.sql_con) 70 self.sql_con = False
71
72 - def process(self):
73 self.sql_con = self.connector.open() 74 cursor = self.sql_con.cursor() 75 cursor.execute(self.sqlquery) 76 columns_description = cursor.description 77 rows = cursor.fetchall() 78 for row in rows: 79 col_count=0 80 d = {} 81 for column in columns_description: 82 d[column[0]] = row[col_count] 83 col_count += 1 84 if d: 85 yield d, 'main' 86 87
88 -def test():
89 from etl_test import etl_test 90 import etl 91 sql_conn = etl.connector.sql_connector('localhost', 5432, 'trunk', 'postgres', 'postgres') 92 query = 'select * from etl_test'# execute the query you wish to 93 test = etl_test.etl_component_test(sql_in(sql_conn, query)) 94 test.check_output([{'id': 1, 'name': 'a'}, {'id': 2, 'name': 'b'}])# output according to the executed query should be written over here. 95 res = test.output() 96 print res
97 98 if __name__ == '__main__': 99 test() 100