This project has retired. For details please refer to its
Attic page.
DatabaseConfig xref
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.chukwa.database;
20
21
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.Path;
24 import java.util.*;
25 import java.io.File;
26 import java.io.FilenameFilter;
27
28 public class DatabaseConfig {
29 private Configuration config = null;
30 public final static long CENTURY = 36500 * 24 * 60 * 60 * 1000L;
31 public final static long DECADE = 3650 * 24 * 60 * 60 * 1000L;
32 public final static long YEAR = 365 * 24 * 60 * 60 * 1000L;
33 public final static long QUARTER = 91250 * 24 * 60 * 60L;
34 public final static long MONTH = 30 * 24 * 60 * 60 * 1000L;
35 public final static long WEEK = 7 * 24 * 60 * 60 * 1000L;
36 public final static long DAY = 24 * 60 * 60 * 1000L;
37 public final static String MDL_XML = "mdl.xml";
38
39 public DatabaseConfig(String path) {
40 Path fileResource = new Path(path);
41 config = new Configuration();
42 config.addResource(fileResource);
43 }
44
45 public DatabaseConfig() {
46 String dataConfig = System.getenv("CHUKWA_CONF_DIR");
47 if (dataConfig == null) {
48 dataConfig = MDL_XML;
49 } else {
50 dataConfig += File.separator + MDL_XML;
51 }
52 Path fileResource = new Path(dataConfig);
53 config = new Configuration();
54 config.addResource(fileResource);
55
56 if (System.getenv("CHUKWA_CONF_DIR") != null) {
57
58
59 File confDir = new File(System.getenv("CHUKWA_CONF_DIR"));
60 File[] confFiles = confDir.listFiles(new FilenameFilter() {
61
62 @Override
63 public boolean accept(File dir, String name) {
64
65
66 return name.endsWith(MDL_XML) && !name.equals(MDL_XML);
67 }
68
69 });
70
71 if (confFiles != null) {
72 for (File confFile : confFiles)
73 config.addResource(new Path(confFile.getAbsolutePath()));
74 }
75 }
76 }
77
78 public String get(String key) {
79 return config.get(key);
80 }
81
82 public void put(String key, String value) {
83 this.config.set(key, value);
84 }
85
86 public Iterator<?> iterator() {
87 return this.config.iterator();
88 }
89
90 public HashMap<String, String> startWith(String key) {
91 HashMap<String, String> transformer = new HashMap<String, String>();
92 Iterator<?> entries = config.iterator();
93 while (entries.hasNext()) {
94 String entry = entries.next().toString();
95 if (entry.startsWith(key)) {
96 String[] metrics = entry.split("=");
97 transformer.put(metrics[0], metrics[1]);
98 }
99 }
100 return transformer;
101 }
102
103 public String[] findTableName(String tableName, long start, long end) {
104 String[] tableNames = null;
105 String tableType = "_week";
106 long now = (new Date()).getTime();
107 long timeWindow = end - start;
108 long partitionSize = WEEK;
109 boolean fallback = true;
110
111 if (config.get("consolidator.table." + tableName) == null) {
112 tableNames = new String[1];
113 tableNames[0] = tableName;
114 return tableNames;
115 }
116 if (timeWindow <= 0) {
117 timeWindow = 1;
118 }
119 if (timeWindow > DECADE) {
120 tableType = "_century";
121 partitionSize = CENTURY;
122 } else if (timeWindow > YEAR) {
123 tableType = "_decade";
124 partitionSize = DECADE;
125 } else if (timeWindow > QUARTER) {
126 tableType = "_year";
127 partitionSize = YEAR;
128 } else if (timeWindow > MONTH) {
129 tableType = "_quarter";
130 partitionSize = QUARTER;
131 } else if (timeWindow > WEEK) {
132 tableType = "_month";
133 partitionSize = MONTH;
134 } else {
135 tableType = "_week";
136 partitionSize = WEEK;
137 }
138
139 long currentPartition = now / partitionSize;
140 long startPartition = start / partitionSize;
141 long endPartition = end / partitionSize;
142 while (fallback && partitionSize != CENTURY * 100) {
143
144
145 if (startPartition + 2 < currentPartition) {
146 fallback = true;
147 if (partitionSize == DAY) {
148 tableType = "_week";
149 partitionSize = WEEK;
150 } else if (partitionSize == WEEK) {
151 tableType = "_month";
152 partitionSize = MONTH;
153 } else if (partitionSize == MONTH) {
154 tableType = "_year";
155 partitionSize = YEAR;
156 } else if (partitionSize == YEAR) {
157 tableType = "_decade";
158 partitionSize = DECADE;
159 } else if (partitionSize == DECADE) {
160 tableType = "_century";
161 partitionSize = CENTURY;
162 } else {
163 partitionSize = 100 * CENTURY;
164 }
165 currentPartition = now / partitionSize;
166 startPartition = start / partitionSize;
167 endPartition = end / partitionSize;
168 } else {
169 fallback = false;
170 }
171 }
172
173 if (startPartition != endPartition) {
174 int delta = (int) (endPartition - startPartition);
175 tableNames = new String[delta + 1];
176 for (int i = 0; i <= delta; i++) {
177 long partition = startPartition + (long) i;
178 tableNames[i] = tableName + "_" + partition + tableType;
179 }
180 } else {
181 tableNames = new String[1];
182 tableNames[0] = tableName + "_" + startPartition + tableType;
183 }
184 return tableNames;
185 }
186
187 public String[] findTableNameForCharts(String tableName, long start, long end) {
188 String[] tableNames = null;
189 String tableType = "_week";
190 long now = (new Date()).getTime();
191 long timeWindow = end - start;
192 if (timeWindow > 60 * 60 * 1000) {
193 timeWindow = timeWindow + 1;
194 }
195 long partitionSize = WEEK;
196 boolean fallback = true;
197
198 if (config.get("consolidator.table." + tableName) == null) {
199 tableNames = new String[1];
200 tableNames[0] = tableName;
201 return tableNames;
202 }
203
204 if (timeWindow <= 0) {
205 timeWindow = 1;
206 }
207 if (timeWindow > DECADE) {
208 tableType = "_decade";
209 partitionSize = CENTURY;
210 } else if (timeWindow > YEAR) {
211 tableType = "_decade";
212 partitionSize = CENTURY;
213 } else if (timeWindow > QUARTER) {
214 tableType = "_decade";
215 partitionSize = DECADE;
216 } else if (timeWindow > MONTH) {
217 tableType = "_year";
218 partitionSize = YEAR;
219 } else if (timeWindow > WEEK) {
220 tableType = "_quarter";
221 partitionSize = QUARTER;
222 } else if (timeWindow > DAY) {
223 tableType = "_month";
224 partitionSize = MONTH;
225 } else {
226 tableType = "_week";
227 partitionSize = WEEK;
228 }
229
230
231 long currentPartition = now / partitionSize;
232 long startPartition = start / partitionSize;
233 long endPartition = end / partitionSize;
234 while (fallback && partitionSize != DECADE * 100) {
235
236
237 if (startPartition + 2 < currentPartition) {
238 fallback = true;
239 if (partitionSize == DAY) {
240 tableType = "_month";
241 partitionSize = MONTH;
242 } else if (partitionSize == WEEK) {
243 tableType = "_quarter";
244 partitionSize = QUARTER;
245 } else if (partitionSize == MONTH) {
246 tableType = "_year";
247 partitionSize = YEAR;
248 } else if (partitionSize == YEAR) {
249 tableType = "_decade";
250 partitionSize = DECADE;
251 } else {
252 partitionSize = CENTURY;
253 }
254 currentPartition = now / partitionSize;
255 startPartition = start / partitionSize;
256 endPartition = end / partitionSize;
257 } else {
258 fallback = false;
259 }
260 }
261
262 if (startPartition != endPartition) {
263 int delta = (int) (endPartition - startPartition);
264 tableNames = new String[delta + 1];
265 for (int i = 0; i <= delta; i++) {
266 long partition = startPartition + (long) i;
267 tableNames[i] = tableName + "_" + partition + tableType;
268 }
269 } else {
270 tableNames = new String[1];
271 tableNames[0] = tableName + "_" + startPartition + tableType;
272 }
273
274 return tableNames;
275 }
276
277 public static void main(String[] args) {
278 DatabaseConfig dbc = new DatabaseConfig();
279 String[] names = dbc.findTableName("system_metrics", 1216140020000L,
280 1218645620000L);
281 for (String n : names) {
282 System.out.println("name:" + n);
283 }
284 }
285 }