1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 Data Map component.
24 """
25
26 from etl.component import component
27 import datetime
28 from etl import tools
29
30 -class map(component):
31 """
32 Data map component.
33 """
34
35 - def __init__(self, map_criteria, preprocess=None, name='component.transfer.map', transformer=None, row_limit=0):
36 """
37 Required Parameters
38 map_criteria : Mapping criteria.
39
40 Extra Parameters
41 name : Name of the component.
42 transformer : Transformer object to transform string data into particular object.
43 preprocess : For initializing the instance of custom variable.
44 """
45
46 super(map, self).__init__(name=name, transformer=transformer, row_limit=row_limit)
47 self._type = 'component.transfer.map'
48 self.map_criteria = map_criteria
49 self.preprocess = preprocess
50
52 res = map(self.map_criteria, self.preprocess, self.name, self.transformer, self.row_limit)
53 return res
54
56 channels = self.input_get()
57 datas = {}
58 if self.preprocess:
59 datas = self.preprocess(self, channels)
60 for channel, trans in channels.items():
61 for iterator in trans:
62 for d in iterator:
63 for channel_dest, channel_value in self.map_criteria.items():
64 result = {}
65 for key, val in channel_value.items():
66 if val:
67 datas['main'] = d
68 datas['tools'] = tools
69 result[key] = eval(val, datas)
70 else:
71 result[key] = val
72 if self.transformer:
73 result = self.transformer.transform(result)
74 yield result, channel_dest
75
77
78 from etl_test import etl_test
79 import etl
80 input_part = [
81 {'id': 1, 'name': 'Fabien', 'country_id': 3},
82 {'id': 2, 'name': 'Luc', 'country_id': 3},
83 {'id': 3, 'name': 'Henry', 'country_id': 1}
84 ]
85 input_cty = [{'id': 1, 'name': 'Belgium'}, {'id': 3, 'name': 'France'}]
86 map_keys = {'main': {
87 'id': "main['id']",
88 'name': "main['id'].upper()",
89 'country': "country_var[main['country_id']]['name']"
90 }}
91
92
93
94
95
96
97
98 test = etl_test.etl_component_test(map(map_keys, None))
99 test.check_input({'partner':input_part, 'countries': input_cty})
100 test.check_output({'partner':input_part, 'countries': input_cty})
101 print test.output()
102
103 if __name__ == '__main__':
104 test()
105