Package etl :: Package component :: Module component'
[hide private]
[frames] | no frames]

Source Code for Module etl.component.component'

  1  # -*- encoding: utf-8 -*- 
  2  ############################################################################## 
  3  # 
  4  #    ETL system- Extract Transfer Load system 
  5  #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved 
  6  #    $Id$ 
  7  # 
  8  #    This program is free software: you can redistribute it and/or modify 
  9  #    it under the terms of the GNU General Public License as published by 
 10  #    the Free Software Foundation, either version 3 of the License, or 
 11  #    (at your option) any later version. 
 12  # 
 13  #    This program is distributed in the hope that it will be useful, 
 14  #    but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  #    GNU General Public License for more details. 
 17  # 
 18  #    You should have received a copy of the GNU General Public License 
 19  #    along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 20  # 
 21  ############################################################################## 
 22  """ 
 23   ETL Component. 
 24   
 25   Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). 
 26   GNU General Public License. 
 27  """ 
 28  import datetime 
 29  from etl import signal 
 30   
 31   
32 -class component(signal):
33 """ 34 Base class of ETL Component. 35 """ 36
37 - def __init__(self, name='', connector=None, transformer=None, row_limit=0):
38 super(component, self).__init__() 39 self._type = 'component' 40 self._cache = {} 41 self.trans_in = [] 42 self.trans_out = [] 43 self.data = {} 44 self.job = False 45 self.generator = None 46 self.name = name 47 self.connector = connector 48 self.transformer = transformer 49 self.row_limit = row_limit 50 self.status = 'open'
51
52 - def __str__(self):
53 res='<Component job="%s" name="%s" type="%s" status="%s"'% (self.job.name, self.name, self._type, self.status) 54 if self.is_start(): 55 res += ' is_start = "True"' 56 if self.is_end(): 57 res += ' is_end = "True"' 58 res += ">" 59 return res
60
61 - def __copy__(self):
62 res = component(name=self.name, transformer=self.transformer) 63 return res 64
65 - def copy(self):
66 res = self.__copy__() 67 res.name += '(copy)' 68 return res
69
70 - def is_start(self):
71 if not len(self.trans_in): 72 return True 73 return False 74
75 - def is_end(self):
76 if not len(self.trans_out): 77 return True 78 return False 79
80 - def pause(self):
81 self.status = 'pause' 82 self.signal('pause', {'date': datetime.datetime.today()}) 83
84 - def stop(self):
85 self.status = 'stop' 86 self.signal('stop', {'date': datetime.datetime.today()}) 87
88 - def end(self):
89 self.status = 'end' 90 self.signal('end', {'date': datetime.datetime.today()})
91
92 - def start(self):
93 self.status = 'start' 94 self.signal('start', {'date': datetime.datetime.today()})
95
96 - def warning(self, message):
97 self.signal('warning', {'message': message})
98
99 - def error(self, message):
100 self.signal('error', {'message': message})
101
102 - def generator_get(self, transition):
103 """ 104 Get generator list of transition. 105 """ 106 if self.generator: 107 return self.generator 108 self.generator = self.process() 109 return self.generator
110
111 - def channel_get(self, trans=None):
112 """ 113 Get channel list of transition. 114 """ 115 #if self.status in ('end','stop') or (trans and trans.type=='trigger' and trans.status in ('end','stop','close')): 116 # return 117 self.data.setdefault(trans, []) 118 self._cache['start_output'] = {trans:False} 119 self._cache['start_input'] = {trans:False} 120 gen = self.generator_get(trans) or None 121 if trans: 122 trans.start() 123 self.start() 124 try: 125 row_count = 0 126 while True: 127 if self.data[trans]: 128 if not self._cache['start_output'][trans]: 129 self._cache['start_output'][trans] = datetime.datetime.today() 130 self.signal('start_output', {'trans': trans, 'date': datetime.datetime.today()}) 131 data = self.data[trans].pop(0) 132 self.signal('send_output', {'trans':trans,'data':data, 'date': datetime.datetime.today()}) 133 yield data 134 continue 135 elif self.data[trans] is None: 136 self.signal('no_input') 137 raise StopIteration 138 139 data, chan = gen.next() 140 row_count += 1 141 if self.row_limit and row_count > self.row_limit: 142 raise StopIteration 143 if data is None: 144 self.signal('no_input') 145 raise StopIteration 146 if self.transformer: 147 data = self.transformer.transform(data) 148 149 if not self._cache['start_input'][trans]: 150 self._cache['start_input'][trans] = datetime.datetime.today() 151 self.signal('start_input', {'trans': trans,'channel':chan, 'date': datetime.datetime.today()}) 152 153 self.signal('get_input', {'trans': trans,'channel':chan,'data':data, 'date': datetime.datetime.today()}) 154 for t, t2 in self.trans_out: 155 if (t == chan) or (not t) or (not chan): 156 self.data.setdefault(t2, []) 157 self.data[t2].append(data) 158 except StopIteration, e: 159 if trans: 160 trans.end() 161 self.end()
162 #except Exception, e: 163 # self.signal('error', {'data': self.data, 'type': 'exception', 'error': str(e)}) 164
165 - def process(self):
166 """ 167 Process method of ETL component. 168 """ 169 pass
170
171 - def get_trigger_data(self, channel, trigger):
172 return None
173
174 - def input_get(self):
175 """ 176 Get input iterator of ETL component. 177 """ 178 result = {} 179 for channel, trans in self.trans_in: 180 result.setdefault(channel, []) 181 if trans == 'trigger': 182 data = trans.source.get_trigger_data(channel, trans.trigger) 183 else: 184 data = trans.source.channel_get(trans) 185 result[channel].append(data) 186 return result
187