1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 Data Join component.
24 """
25
26 from etl.component import component
27 from etl.component.transform import map
28 import datetime
29 from etl import tools
30
32 """
33 Data Join component.
34 """
35
36 - def __init__(self, map_criteria, join_keys={}, name='component.transfer.join', transformer=None, row_limit=0):
37
38 """
39 Required Parameters
40 map_criteria : Mapping criteria.
41
42 Extra Parameters
43 name : Name of the component.
44 join_keys :
45 transformer : Transformer object to transform string data into particular object.
46 """
47 super(map, self).__init__(name=name, transformer=transformer, row_limit=0)
48 self._type = 'component.transfer.join'
49 self.map_criteria = map_criteria
50 self.join_keys = join_keys
51
52 def preprocess(self, channels):
53 res = {}
54 for chn in join_keys:
55 cdict = {}
56 for iterator in channels[chn]:
57 for d in iterator:
58 cdict[d[join_keys[chn]]] = d
59 res[chn] = cdict
60 return res
61 self.preprocess = preprocess
62
64 res = join(self.map_criteria, self.join_keys, self.name, self.transformer, self.row_limit)
65 return res
66
70
71
73 from etl_test import etl_test
74 import etl
75 input_part = [
76 {'id': 1, 'name': 'Fabien', 'country_id': 3},
77 {'id': 2, 'name': 'Luc', 'country_id': 3},
78 {'id': 3, 'name': 'Henry', 'country_id': 1}
79 ]
80 input_cty = [{'id': 1, 'name': 'Belgium'}, {'id': 3, 'name': 'France'}]
81 map_keys = {'main': {
82 'id': "main['id']",
83 'name': "main['id'].upper()",
84 'country': "country[main['country_id']]['name']"
85 }}
86 join_keys = {
87 'country': 'id'
88 }
89 test = etl_test.etl_component_test(join(map_keys, join_keys))
90 test.check_input({'partner':input_part, 'countries': input_cty})
91 print test.output()
92
93 if __name__ == '__main__':
94 test()
95