1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 ETL Component.
24
25 Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
26 GNU General Public License.
27 """
28 import datetime
29 from etl import signal
30
31
33 """
34 Base class of ETL Component.
35 """
36
37 - def __init__(self, name='', connector=None, transformer=None, row_limit=0):
51
53 res='<Component job="%s" name="%s" type="%s" status="%s"'% (self.job.name, self.name, self._type, self.status)
54 if self.is_start():
55 res += ' is_start = "True"'
56 if self.is_end():
57 res += ' is_end = "True"'
58 res += ">"
59 return res
60
62 res = component(name=self.name, transformer=self.transformer)
63 return res
64
66 res = self.__copy__()
67 res.name += '(copy)'
68 return res
69
71 if not len(self.trans_in):
72 return True
73 return False
74
76 if not len(self.trans_out):
77 return True
78 return False
79
81 self.status = 'pause'
82 self.signal('pause', {'date': datetime.datetime.today()})
83
85 self.status = 'stop'
86 self.signal('stop', {'date': datetime.datetime.today()})
87
91
95
97 self.signal('warning', {'message': message})
98
99 - def error(self, message):
100 self.signal('error', {'message': message})
101
103 """
104 Get generator list of transition.
105 """
106 if self.generator:
107 return self.generator
108 self.generator = self.process()
109 return self.generator
110
112 """
113 Get channel list of transition.
114 """
115
116
117 self.data.setdefault(trans, [])
118 self._cache['start_output'] = {trans:False}
119 self._cache['start_input'] = {trans:False}
120 gen = self.generator_get(trans) or None
121 if trans:
122 trans.start()
123 self.start()
124 try:
125 row_count = 0
126 while True:
127 if self.data[trans]:
128 if not self._cache['start_output'][trans]:
129 self._cache['start_output'][trans] = datetime.datetime.today()
130 self.signal('start_output', {'trans': trans, 'date': datetime.datetime.today()})
131 data = self.data[trans].pop(0)
132 self.signal('send_output', {'trans':trans,'data':data, 'date': datetime.datetime.today()})
133 yield data
134 continue
135 elif self.data[trans] is None:
136 self.signal('no_input')
137 raise StopIteration
138
139 data, chan = gen.next()
140 row_count += 1
141 if self.row_limit and row_count > self.row_limit:
142 raise StopIteration
143 if data is None:
144 self.signal('no_input')
145 raise StopIteration
146 if self.transformer:
147 data = self.transformer.transform(data)
148
149 if not self._cache['start_input'][trans]:
150 self._cache['start_input'][trans] = datetime.datetime.today()
151 self.signal('start_input', {'trans': trans,'channel':chan, 'date': datetime.datetime.today()})
152
153 self.signal('get_input', {'trans': trans,'channel':chan,'data':data, 'date': datetime.datetime.today()})
154 for t, t2 in self.trans_out:
155 if (t == chan) or (not t) or (not chan):
156 self.data.setdefault(t2, [])
157 self.data[t2].append(data)
158 except StopIteration, e:
159 if trans:
160 trans.end()
161 self.end()
162
163
164
166 """
167 Process method of ETL component.
168 """
169 pass
170
173
187