This project has retired. For details please refer to its Attic page.
Swimlanes xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18   
19  package org.apache.hadoop.chukwa.analysis.salsa.visualization;
20  
21  import prefuse.data.io.sql.*;
22  import prefuse.data.expression.parser.*;
23  import prefuse.data.expression.*;
24  import prefuse.data.column.*;
25  import prefuse.data.query.*;
26  import prefuse.data.*;
27  import prefuse.action.*;
28  import prefuse.action.layout.*;
29  import prefuse.action.assignment.*;
30  import prefuse.visual.expression.*;
31  import prefuse.visual.*;
32  import prefuse.render.*;
33  import prefuse.util.collections.*;
34  import prefuse.util.*;
35  import prefuse.*;
36  
37  import org.apache.hadoop.chukwa.hicc.OfflineTimeHandler;
38  import org.apache.hadoop.chukwa.hicc.TimeHandler;
39  import org.apache.hadoop.chukwa.util.DatabaseWriter;
40  import org.apache.hadoop.chukwa.database.Macro;
41  import org.apache.hadoop.chukwa.util.XssFilter;
42  
43  import org.apache.commons.logging.Log;
44  import org.apache.commons.logging.LogFactory;
45  
46  import javax.servlet.http.*;
47  
48  import java.sql.*;
49  import java.util.*;
50  
51  import java.awt.Font;
52  import java.awt.geom.Rectangle2D;
53  
54  /**
55   * Static image generation for Swimlanes visualization for scalable 
56   * rendering on front-end client (web-browser)
57   * Handles database data retrieval, transforming data to form for 
58   * visualization elements, and initializing and calling visualization
59   * elements
60   */
61  public class Swimlanes {
62  
63    private static Log log = LogFactory.getLog(Swimlanes.class);
64  
65    int SIZE_X=1600, SIZE_Y=1600;
66    final int [] BORDER = {50,50,50,50};
67    final int LEGEND_X_OFFSET = 50;
68    final int LEGEND_Y_OFFSET = 25;
69    final int LEGEND_TEXT_OFFSET = 20;
70    final int LEGEND_FONT_SIZE = 18;
71    final int AXIS_NAME_FONT_SIZE = 24;
72  
73    protected boolean offline_use = true;
74    protected HttpServletRequest request;
75    
76    protected String abc;
77  
78    /**
79     * Modifier for generic Swimlanes plots to plot shuffle, sort, and reducer
80     * states of same reduce on same line 
81     */
82    protected static class MapReduceSwimlanes {
83      protected Table plot_tab;
84      protected HashMap<String, ArrayList<Tuple> > reducepart_hash;
85      protected boolean collate_reduces = false;
86      
87      public MapReduceSwimlanes() {
88        this.plot_tab = new Table();
89        this.plot_tab.addColumn("ycoord",float.class);
90        this.plot_tab.addColumn("state_name",String.class);
91        this.plot_tab.addColumn("hostname",String.class);
92        this.plot_tab.addColumn("friendly_id",String.class);
93        this.plot_tab.addColumn(START_FIELD_NAME,double.class);
94        this.plot_tab.addColumn(END_FIELD_NAME,double.class);
95        this.plot_tab.addColumn(PolygonRenderer.POLYGON,float[].class);
96        this.reducepart_hash = new HashMap<String, ArrayList<Tuple> >();
97      }
98      
99      public void populateTable_OneLinePerState(Table orig_tab) {
100       IntIterator rownumiter;
101       int newrownum, origrownum;
102       rownumiter = orig_tab.rows(); // iterate over everything
103       while (rownumiter.hasNext()) {
104         origrownum = ((Integer)rownumiter.next()).intValue();
105         newrownum = this.plot_tab.addRow();
106         this.plot_tab.set(newrownum, "state_name", orig_tab.getString(origrownum, "state_name"));
107         this.plot_tab.set(newrownum, "ycoord", orig_tab.getInt(origrownum, "seqno"));
108         this.plot_tab.set(newrownum,"hostname",orig_tab.getString(origrownum,"hostname"));
109         this.plot_tab.set(newrownum,"friendly_id",orig_tab.getString(origrownum,"friendly_id"));
110         this.plot_tab.set(newrownum,START_FIELD_NAME, orig_tab.getDouble(origrownum,START_FIELD_NAME));
111         this.plot_tab.set(newrownum,END_FIELD_NAME, orig_tab.getDouble(origrownum,END_FIELD_NAME));
112       }      
113     }
114     
115     public void populateTable_CollateReduces(Table orig_tab) {
116       IntIterator rownumiter;
117       int newrownum, origrownum;
118       
119       this.collate_reduces = true;
120       
121       // add maps normally
122       rownumiter = orig_tab.rows(
123         (Predicate) ExpressionParser.parse("[state_name] == 'map' " + 
124           "OR [state_name] == 'shuffle_local' " + 
125           "OR [state_name] == 'shuffle_remote'")
126       );
127       
128       while (rownumiter.hasNext()) {
129         origrownum = ((Integer)rownumiter.next()).intValue();
130         newrownum = this.plot_tab.addRow();
131         this.plot_tab.set(newrownum, "state_name", orig_tab.getString(origrownum, "state_name"));
132         this.plot_tab.set(newrownum, "ycoord", orig_tab.getInt(origrownum, "seqno"));
133         this.plot_tab.set(newrownum,"hostname",orig_tab.getString(origrownum,"hostname"));
134         this.plot_tab.set(newrownum,"friendly_id",orig_tab.getString(origrownum,"friendly_id"));
135         this.plot_tab.set(newrownum,START_FIELD_NAME, orig_tab.getDouble(origrownum,START_FIELD_NAME));
136         this.plot_tab.set(newrownum,END_FIELD_NAME, orig_tab.getDouble(origrownum,END_FIELD_NAME));
137       }
138 
139       // special breakdown for reduces
140       IntIterator rownumiter3 = orig_tab.rows(
141         (Predicate) ExpressionParser.parse("[state_name] == 'reduce_reducer' " +
142           "OR [state_name] == 'reduce_shufflewait' " + 
143           "OR [state_name] == 'reduce_sort' " + 
144           "OR [state_name] == 'reduce'")
145       );
146       
147       ArrayList<Tuple> tuple_array;
148       while (rownumiter3.hasNext()) {
149         origrownum = ((Integer)rownumiter3.next()).intValue();
150         if (orig_tab.getString(origrownum,"state_name").equals("reduce")) {
151           continue; // do NOT add reduces
152         }
153         String curr_reduce = orig_tab.getString(origrownum, "friendly_id");
154         newrownum = this.plot_tab.addRow();
155         
156         this.plot_tab.set(newrownum, "state_name", orig_tab.getString(origrownum, "state_name"));
157         this.plot_tab.set(newrownum, "ycoord", orig_tab.getInt(origrownum, "seqno"));
158         this.plot_tab.set(newrownum,"hostname",orig_tab.getString(origrownum,"hostname"));
159         this.plot_tab.set(newrownum,"friendly_id",orig_tab.getString(origrownum,"friendly_id"));        
160         this.plot_tab.set(newrownum,START_FIELD_NAME, orig_tab.getDouble(origrownum,START_FIELD_NAME));
161         this.plot_tab.set(newrownum,END_FIELD_NAME, orig_tab.getDouble(origrownum,END_FIELD_NAME));
162         
163         tuple_array = this.reducepart_hash.get(curr_reduce);
164         if (tuple_array == null) {
165           tuple_array = new ArrayList<Tuple>();
166           tuple_array.add(this.plot_tab.getTuple(newrownum));
167           this.reducepart_hash.put(curr_reduce, tuple_array);
168         } else {
169           tuple_array.add(this.plot_tab.getTuple(newrownum));
170         }
171       }  
172     }
173     
174     public void populateTable_MapsReducesOnly(Table orig_tab) {
175       IntIterator rownumiter;
176       int newrownum, origrownum;
177       rownumiter = orig_tab.rows(
178         (Predicate) ExpressionParser.parse("[state_name] == 'map' OR [state_name] == 'reduce'")
179       );
180       while (rownumiter.hasNext()) {
181         origrownum = ((Integer)rownumiter.next()).intValue();
182         newrownum = this.plot_tab.addRow();
183         this.plot_tab.set(newrownum, "state_name", orig_tab.getString(origrownum, "state_name"));
184         this.plot_tab.set(newrownum, "ycoord", orig_tab.getInt(origrownum, "seqno"));
185         this.plot_tab.set(newrownum,"hostname",orig_tab.getString(origrownum,"hostname"));
186         this.plot_tab.set(newrownum,"friendly_id",orig_tab.getString(origrownum,"friendly_id"));        
187         this.plot_tab.set(newrownum,START_FIELD_NAME, orig_tab.getDouble(origrownum,START_FIELD_NAME));
188         this.plot_tab.set(newrownum,START_FIELD_NAME, orig_tab.getDouble(origrownum,END_FIELD_NAME));
189       }
190     }
191     
192     /**
193      * Reassigns Y coord values to group by state
194      */
195     public void groupByState() {
196       int counter, rownum;
197       int rowcount = this.plot_tab.getRowCount();
198       HashSet<String> states = new HashSet<String>();
199       String curr_state = null;
200       Iterator<String> state_iter;
201       IntIterator rownumiter;
202       
203       for (int i = 0; i < rowcount; i++) {
204         states.add(this.plot_tab.getString(i,"state_name"));
205       }
206      
207       state_iter = states.iterator();
208       counter = 1;
209       while (state_iter.hasNext()) {
210         curr_state = state_iter.next();
211         
212         if (this.collate_reduces && ((curr_state.equals("reduce_reducer") || curr_state.equals("reduce_sort")))) {
213           continue;
214         }
215         rownumiter = this.plot_tab.rows(
216           (Predicate) ExpressionParser.parse("[state_name] == '"+curr_state+"'")
217         );
218         if (this.collate_reduces && curr_state.equals("reduce_shufflewait")) {
219           while (rownumiter.hasNext()) {
220             rownum = ((Integer)rownumiter.next()).intValue();
221             this.plot_tab.setFloat(rownum,"ycoord",(float)counter);
222             
223             ArrayList<Tuple> alt = this.reducepart_hash.get(this.plot_tab.getString(rownum,"friendly_id"));
224             Object [] tarr = alt.toArray();
225             for (int i = 0; i < tarr.length; i++) ((Tuple)tarr[i]).setFloat("ycoord",(float)counter);
226             counter++;            
227           }
228         } else {
229           while (rownumiter.hasNext()) {
230             rownum = ((Integer)rownumiter.next()).intValue();
231             this.plot_tab.setFloat(rownum,"ycoord",(float)counter);
232             counter++;
233           }          
234         }
235       }
236     }
237     
238     public void groupByStartTime() {
239       int counter, rownum;
240       String curr_state = null;
241       IntIterator rownumiter;
242      
243       rownumiter = this.plot_tab.rowsSortedBy(START_FIELD_NAME, true);
244      
245       counter = 1;
246       while (rownumiter.hasNext()) {
247         rownum = ((Integer)rownumiter.next()).intValue();
248         curr_state = this.plot_tab.getString(rownum, "state_name");        
249 
250         if (this.collate_reduces && curr_state.equals("reduce_shufflewait")) {
251           this.plot_tab.setFloat(rownum,"ycoord",(float)counter);
252           ArrayList<Tuple> alt = this.reducepart_hash.get(this.plot_tab.getString(rownum,"friendly_id"));
253           Object [] tarr = alt.toArray();
254           for (int i = 0; i < tarr.length; i++) ((Tuple)tarr[i]).setFloat("ycoord",(float)counter);
255           counter++;   
256         } else if (!curr_state.equals("reduce_sort") && !curr_state.equals("reduce_reducer")) {
257           this.plot_tab.setFloat(rownum,"ycoord",(float)counter);          
258           counter++;
259         }
260       }
261     }
262     
263     public void groupByEndTime() {
264       int counter, rownum;
265       String curr_state = null;
266       IntIterator rownumiter;
267      
268       rownumiter = this.plot_tab.rowsSortedBy(END_FIELD_NAME, true);
269       counter = 1;
270       while (rownumiter.hasNext()) {
271         rownum = ((Integer)rownumiter.next()).intValue();
272         curr_state = this.plot_tab.getString(rownum, "state_name");        
273 
274         if (this.collate_reduces && curr_state.equals("reduce_reducer")) {
275           this.plot_tab.setFloat(rownum,"ycoord",(float)counter);
276           ArrayList<Tuple> alt = this.reducepart_hash.get(this.plot_tab.getString(rownum,"friendly_id"));
277           Object [] tarr = alt.toArray();
278           for (int i = 0; i < tarr.length; i++) ((Tuple)tarr[i]).setFloat("ycoord",(float)counter);
279           counter++;
280         } else if (!curr_state.equals("reduce_sort") && !curr_state.equals("reduce_shufflewait")) {
281           this.plot_tab.setFloat(rownum,"ycoord",(float)counter);          
282           counter++;
283         }
284       }
285     }    
286     
287     public VisualTable addToVisualization(Visualization viz, String groupname) {
288       return viz.addTable(groupname, this.plot_tab);
289     }
290   }
291 
292   /**
293    * Provide constant mapping between state names and colours
294    * so that even if particular states are missing, the colours are fixed
295    * for each state
296    */
297   public static class SwimlanesStatePalette {
298     protected final String [] states = {"map","reduce","reduce_shufflewait","reduce_sort","reduce_reducer","shuffle"};
299     HashMap<String,Integer> colourmap; 
300     protected int [] palette;
301     public SwimlanesStatePalette() {
302       palette = ColorLib.getCategoryPalette(states.length);
303       colourmap = new HashMap<String,Integer>();
304       for (int i = 0; i < states.length; i++) {
305         colourmap.put(states[i], Integer.valueOf(palette[i]));
306       }
307     }
308     public int getColour(String state_name) {
309       Integer val = colourmap.get(state_name);
310       if (val == null) {
311         return ColorLib.color(java.awt.Color.BLACK);
312       } else {
313         return val.intValue();
314       }
315     }
316     public int getNumStates() {
317       return this.states.length;
318     }
319     public String [] getStates() {
320       return this.states.clone();
321     }
322   }
323 
324   /**
325    * Provides convenient rescaling of raw values to be plotted to
326    * actual pixels for plotting on image
327    */
328   public static class CoordScaler {
329     double x_pixel_size, y_pixel_size;
330     double x_max_value, y_max_value, x_min_value, y_min_value;
331     double x_start, y_start;
332     
333     public CoordScaler() {
334       this.x_pixel_size = 0.0;
335       this.y_pixel_size = 0.0;
336       this.x_max_value = 1.0;
337       this.y_max_value = 1.0;
338       this.x_min_value = 0.0;
339       this.y_min_value = 0.0;
340       this.x_start = 0.0;
341       this.y_start = 0.0;
342     }
343     public void set_pixel_start(double x, double y) {
344       this.x_start = x;
345       this.y_start = y;
346     }
347     public void set_pixel_size(double x, double y) {
348       this.x_pixel_size = x;
349       this.y_pixel_size = y;
350     }
351     public void set_value_ranges(double x_min, double y_min, double x_max, double y_max) {
352       this.x_max_value = x_max;
353       this.y_max_value = y_max;
354       this.x_min_value = x_min;
355       this.y_min_value = y_min;
356     }
357     public double get_x_coord(double x_value) {
358       return x_start+(((x_value - x_min_value) / (x_max_value-x_min_value)) * x_pixel_size);
359     }
360     public double get_y_coord(double y_value) {
361       // this does "inverting" to shift the (0,0) point from top-right to bottom-right
362       return y_start+(y_pixel_size - ((((y_value - y_min_value) / (y_max_value-y_min_value)) * y_pixel_size)));
363     }
364   }
365 
366   /**
367    * Prefuse action for plotting a line for each state
368    */
369   public static class SwimlanesStateAction extends GroupAction {
370     
371     protected CoordScaler cs;
372     
373     public SwimlanesStateAction() {
374       super();
375     }
376     
377     public SwimlanesStateAction(String group, CoordScaler cs) {
378       super(group);
379       this.cs = cs;
380     }
381     
382     public void run (double frac) {
383       VisualItem item = null;
384       SwimlanesStatePalette pal = new SwimlanesStatePalette();
385       
386       Iterator<?> curr_group_items = this.m_vis.items(this.m_group);
387           
388       while (curr_group_items.hasNext()) {
389         item = (VisualItem) curr_group_items.next();
390         
391         double start_time = item.getDouble(START_FIELD_NAME);
392         double finish_time = item.getDouble(END_FIELD_NAME);        
393         item.setShape(Constants.POLY_TYPE_LINE);
394         item.setX(0.0);
395         item.setY(0.0);        
396         
397         float [] coords = new float[4];
398         coords[0] = (float) cs.get_x_coord(start_time);
399         coords[1] = (float) cs.get_y_coord((double)item.getInt("ycoord"));
400         coords[2] = (float) cs.get_x_coord(finish_time);
401         coords[3] = (float) cs.get_y_coord((double)item.getInt("ycoord"));
402 
403         item.set(VisualItem.POLYGON,coords);
404         item.setStrokeColor(pal.getColour(item.getString("state_name")));
405       }
406     }    
407   } // SwimlanesStateAction
408 
409   // keys that need to be filled:
410   // period (last1/2/3/6/12/24hr,last7d,last30d), time_type (range/last), start, end
411   protected HashMap<String, String> param_map;
412   
413   protected String cluster;
414   protected String timezone;
415   protected String shuffle_option;
416   protected final String table = "mapreduce_fsm";
417   protected boolean plot_legend = true;
418   protected String jobname = null;
419   
420   protected Display dis;
421   protected Visualization viz;
422   
423   protected Rectangle2D dataBound = new Rectangle2D.Double();
424   protected Rectangle2D xlabBound = new Rectangle2D.Double();
425   protected Rectangle2D ylabBound = new Rectangle2D.Double();
426   protected Rectangle2D labelBottomBound = new Rectangle2D.Double();
427   
428   static final String START_FIELD_NAME = "start_time_num";
429   static final String END_FIELD_NAME = "finish_time_num";
430   
431   /* Different group names allow control of what Renderers to use */
432   final String maingroup = "Job";
433   final String othergroup = "Misc";
434   final String labelgroup = "Label";
435   final String legendgroup = "Legend";
436   final String legendshapegroup = "LegendShape";
437   
438   public Swimlanes() {
439     this.cluster = "";
440     this.timezone = "";
441     this.shuffle_option = "";
442     param_map = new HashMap<String, String>();
443   }
444   
445   /**
446    * Constructor for Swimlanes visualization object
447    * @param timezone Timezone string from environment
448    * @param cluster Cluster name from environment
449    * @param event_type Whether to display shuffles or not
450    * @param valmap HashMap of key/value pairs simulating parameters from a HttpRequest
451    */
452   public Swimlanes
453     (String timezone, String cluster, String event_type, 
454      HashMap<String, String> valmap) 
455   {
456     this.cluster = cluster;
457     if (timezone != null) {
458       this.timezone = timezone;
459     } else {
460       this.timezone = null;
461     }
462     this.shuffle_option = event_type;
463     
464     /* This should "simulate" an HttpServletRequest
465      * Need to have "start" and "end" in seconds since Epoch
466      */
467     this.param_map = valmap;
468   }
469   
470   public Swimlanes
471     (String timezone, String cluster, String event_type, 
472      HashMap<String, String> valmap, int width, int height) 
473   {
474     this.cluster = cluster;
475     if (timezone != null) {
476       this.timezone = timezone;
477     } else {
478       this.timezone = null;
479     }
480     this.shuffle_option = event_type;
481     
482     /* This should "simulate" an HttpServletRequest
483      * Need to have "start" and "end" in seconds since Epoch
484      */
485     this.param_map = valmap; 
486     
487     this.SIZE_X = width;
488     this.SIZE_Y = height;
489   }
490   
491   public Swimlanes
492     (String timezone, String cluster, String event_type, 
493      HashMap<String, String> valmap, int width, int height,
494      String legend_opt) 
495   {
496     this.cluster = cluster;
497     if (timezone != null) {
498       this.timezone = timezone;
499     } else {
500       this.timezone = null;
501     }
502     this.shuffle_option = event_type;
503     
504     /* This should "simulate" an HttpServletRequest
505      * Need to have "start" and "end" in seconds since Epoch
506      */
507     this.param_map = valmap;
508     
509     this.SIZE_X = width;
510     this.SIZE_Y = height;
511     
512     if (legend_opt.equals("nolegend")) {
513       this.plot_legend = false;
514     }
515     
516   }
517   
518   public Swimlanes(HttpServletRequest request) {
519     XssFilter xf = new XssFilter(request);
520     this.offline_use = false;
521     this.request = request;
522     HttpSession session = request.getSession();
523     this.cluster = session.getAttribute("cluster").toString();
524       String evt_type = xf.getParameter("event_type");
525     if (evt_type != null) {
526       this.shuffle_option = evt_type;
527     } else {
528       this.shuffle_option = "noshuffle";
529     }
530     this.timezone = session.getAttribute("time_zone").toString();
531   }
532   
533   /**
534    * Set job ID to filter results on
535    * Call before calling @see #run
536    * @param s job name
537    */
538   public void setJobName(String s) {
539     this.jobname = s;
540   }
541 
542   /**
543    * Set dimensions of image to be generated
544    * Call before calling @see #run
545    * @param width image width in pixels
546    * @param height image height in pixels
547    */  
548   public void setDimensions(int width, int height) {
549     this.SIZE_X=width;
550     this.SIZE_Y=height;
551   }
552   
553   /**
554    * Specify whether to print legend of states
555    * Advisable to not print legend for excessively small images since
556    * legend has fixed point size
557    * Call before calling @see #run
558    * @param legendopt parameter to turn on legends
559    */
560   public void setLegend(boolean legendopt) {
561     if (legendopt) {
562       this.plot_legend = true;
563     } else {
564       this.plot_legend = false;
565     }
566   }
567   
568   /**
569    * Generates image in specified format, and writes image as binary
570    * output to supplied output stream 
571    * @param output output stream of image
572    * @param img_fmt image format
573    * @param scale image scaling factor
574    * @return true if image is saved
575    */
576   public boolean getImage(java.io.OutputStream output, String img_fmt, double scale) {
577     dis = new Display(this.viz);
578     dis.setSize(SIZE_X,SIZE_Y);
579     dis.setHighQuality(true);
580     dis.setFont(new Font(Font.SANS_SERIF,Font.PLAIN,24));
581     return dis.saveImage(output, img_fmt, scale);
582   } 
583   
584   /**
585    * Adds a column to given table by converting timestamp to long with
586    * seconds since epoch, and adding milliseconds from additional column
587    * in original table
588    *
589    * @param origTable Table to add to
590    * @param srcFieldName Name of column containing timestamp
591    * @param srcMillisecondFieldName Name of column containing millisecond value of time 
592    * @param dstFieldName Name of new column to add
593    * 
594    * @return Modified table with added column
595    */
596   protected Table addTimeCol 
597     (Table origTable, String srcFieldName, 
598      String srcMillisecondFieldName, String dstFieldName)
599   {
600     origTable.addColumn(dstFieldName, long.class);
601     
602     int total_rows = origTable.getRowCount();
603     for (int curr_row_num = 0; curr_row_num < total_rows; curr_row_num++) {
604       origTable.setLong(curr_row_num, dstFieldName, 
605         ((Timestamp)origTable.get(curr_row_num, srcFieldName)).getTime() + 
606         origTable.getLong(curr_row_num, srcMillisecondFieldName)
607       );
608     }
609     
610     return origTable;
611   }
612   
613   /**
614    * Adds a column with number of seconds of timestamp elapsed since lowest
615    * start time; allows times to be plotted as a delta of the start time
616    * 
617    * @param origTable Table to add column to
618    * @param srcFieldName Name of column containing timestamp
619    * @param srcMillisecondFieldName Name of column containing millisecond value of time 
620    * @param dstFieldName Name of new column to add
621    *   
622    * @return Modified table with added column
623    */
624   protected Table addTimeOffsetCol
625     (Table origTable, String srcFieldName,
626      String srcMillisecondFieldName, String dstFieldName,
627      long timeOffset) 
628   {
629     Table newtable = addTimeCol(origTable, srcFieldName, 
630       srcMillisecondFieldName, dstFieldName + "_fulltime");
631       
632     ColumnMetadata dstcol = newtable.getMetadata(dstFieldName + "_fulltime");
633     long mintime = newtable.getLong(dstcol.getMinimumRow(), dstFieldName + "_fulltime");
634     
635     if (timeOffset == 0) {
636       newtable.addColumn(dstFieldName, "ROUND((["+dstFieldName+"_fulltime] - " + mintime +"L) / 1000L)");
637     } else {
638       newtable.addColumn(dstFieldName, "ROUND((["+dstFieldName+"_fulltime] - " + timeOffset +"L) / 1000L)");      
639     }
640     
641     return newtable;
642   }
643   
644   protected void setupRenderer() {
645     this.viz.setRendererFactory(new RendererFactory(){
646       AbstractShapeRenderer sr = new ShapeRenderer();
647       ShapeRenderer sr_big = new ShapeRenderer(20);
648       Renderer arY = new AxisRenderer(Constants.LEFT, Constants.TOP);
649       Renderer arX = new AxisRenderer(Constants.CENTER, Constants.BOTTOM);
650       PolygonRenderer pr = new PolygonRenderer(Constants.POLY_TYPE_LINE);
651       LabelRenderer lr = new LabelRenderer("label");
652       LabelRenderer lr_legend = new LabelRenderer("label");
653       
654       public Renderer getRenderer(VisualItem item) {
655         lr.setHorizontalAlignment(Constants.CENTER);
656         lr.setVerticalAlignment(Constants.TOP);
657         lr_legend.setHorizontalAlignment(Constants.LEFT);
658         lr_legend.setVerticalAlignment(Constants.CENTER);
659         
660         if (item.isInGroup("ylab")) {
661           return arY;
662         } else if (item.isInGroup("xlab")) {
663           return arX;
664         } else if (item.isInGroup(maingroup)) {
665           return pr;
666         } else if (item.isInGroup(labelgroup)) {
667           return lr;
668         } else if (item.isInGroup(legendgroup)) {
669           return lr_legend;
670         } else if (item.isInGroup(legendshapegroup)) {
671           return sr_big;
672         } else {
673           return sr;
674         }
675       }
676     });
677   }
678   
679   // setup columns: add additional time fields
680   protected Table setupDataTable() {
681     Table res_tab = this.getData();    
682     if (res_tab == null) {
683         return res_tab;
684     }
685     
686     res_tab.addColumn("seqno","ROW()");
687     res_tab = addTimeOffsetCol(res_tab, "start_time", "start_time_millis", START_FIELD_NAME, 0);    
688     ColumnMetadata dstcol = res_tab.getMetadata(START_FIELD_NAME);
689     long mintime = ((Timestamp)res_tab.get(dstcol.getMinimumRow(), "start_time")).getTime();
690     res_tab = addTimeOffsetCol(res_tab, "finish_time", "finish_time_millis", END_FIELD_NAME, mintime);    
691     res_tab.addColumn(PolygonRenderer.POLYGON,float[].class);
692     
693     log.debug("After adding seqno: #cols: " + res_tab.getColumnCount() + "; #rows: " + res_tab.getRowCount());
694     
695     return res_tab;
696   }
697   
698   protected void addAxisNames() {
699     Table textlabels_table = new Table();
700     textlabels_table.addColumn("label",String.class);
701     textlabels_table.addColumn("type",String.class);
702     textlabels_table.addRow();
703     textlabels_table.setString(0,"label","Time/s");
704     textlabels_table.setString(0,"type","xaxisname");
705     
706     VisualTable textlabelsviz = this.viz.addTable(labelgroup, textlabels_table);
707     textlabelsviz.setX(0,SIZE_X/2d);
708     textlabelsviz.setY(0,SIZE_Y - BORDER[2] + (BORDER[2]*0.1));
709     textlabelsviz.setTextColor(0,ColorLib.color(java.awt.Color.GRAY));
710     textlabelsviz.setFont(0,new Font(Font.SANS_SERIF,Font.PLAIN,AXIS_NAME_FONT_SIZE));
711   }
712   
713   protected void addLegend() {
714     SwimlanesStatePalette ssp = new SwimlanesStatePalette();
715     
716     Table shapes_table = new Table();
717     shapes_table.addColumn(VisualItem.X,float.class);
718     shapes_table.addColumn(VisualItem.Y,float.class);
719     
720     Table legend_labels_table = new Table();
721     legend_labels_table.addColumn("label",String.class);
722     
723     // add labels
724     int num_states = ssp.getNumStates();
725     String [] state_names = ssp.getStates();
726     legend_labels_table.addRows(num_states);
727     shapes_table.addRows(num_states);
728     for (int i = 0; i < num_states; i++) {
729       legend_labels_table.setString(i,"label",state_names[i]);
730     }
731     
732     // add legend shapes, manipulate visualitems to set colours
733     VisualTable shapes_table_viz = viz.addTable(legendshapegroup, shapes_table);
734     float start_x = BORDER[0] + LEGEND_X_OFFSET;
735     float start_y = BORDER[1] + LEGEND_Y_OFFSET;
736     float incr = (float) 30.0;
737     for (int i = 0; i < num_states; i++) {
738       shapes_table_viz.setFillColor(i, ssp.getColour(state_names[i]));
739       shapes_table_viz.setFloat(i, VisualItem.X, start_x);
740       shapes_table_viz.setFloat(i, VisualItem.Y, start_y + (i * incr));
741     }
742     
743     // add legend labels, manipulate visualitems to set font
744     VisualTable legend_labels_table_viz = this.viz.addTable(legendgroup, legend_labels_table);
745     for (int i = 0; i < num_states; i++) {
746       legend_labels_table_viz.setFloat(i, VisualItem.X, start_x + LEGEND_TEXT_OFFSET);
747       legend_labels_table_viz.setFloat(i, VisualItem.Y, start_y + (i * incr));
748       legend_labels_table_viz.setTextColor(i,ColorLib.color(java.awt.Color.BLACK));
749       legend_labels_table_viz.setFont(i,new Font(Font.SANS_SERIF,Font.PLAIN,LEGEND_FONT_SIZE));
750     }
751     
752   }
753   
754   public void run() {
755 
756     // setup bounds
757     this.dataBound.setRect(BORDER[0],BORDER[1],SIZE_X-BORDER[2]-BORDER[0],SIZE_Y-BORDER[3]-BORDER[1]);
758     this.xlabBound.setRect(BORDER[0],BORDER[1],SIZE_X-BORDER[2]-BORDER[0],SIZE_Y-BORDER[3]-BORDER[1]);
759     this.ylabBound.setRect(BORDER[0],BORDER[1],SIZE_X-BORDER[2]-BORDER[0],SIZE_Y-BORDER[3]-BORDER[1]);
760     this.labelBottomBound.setRect(BORDER[0],SIZE_X-BORDER[2],SIZE_Y-BORDER[0]-BORDER[1],BORDER[3]);
761     
762     // setup visualization
763     this.viz = new Visualization();
764     this.setupRenderer();
765     
766     // add table to visualization
767     Table raw_data_tab = this.setupDataTable();
768     MapReduceSwimlanes mrs = new MapReduceSwimlanes();
769     mrs.populateTable_CollateReduces(raw_data_tab);
770     mrs.groupByState();
771     VisualTable maindatatable = mrs.addToVisualization(this.viz, maingroup);
772         
773     addAxisNames();
774     if (plot_legend) {
775       addLegend();
776     }
777 
778     // plot swimlanes lines: setup axes, call custom action
779     ActionList draw = new ActionList();
780     {
781       // setup axes
782       AxisLayout xaxis = new AxisLayout(maingroup, START_FIELD_NAME, Constants.X_AXIS, VisiblePredicate.TRUE);
783       AxisLayout yaxis = new AxisLayout(maingroup, "ycoord", Constants.Y_AXIS, VisiblePredicate.FALSE);    
784       xaxis.setLayoutBounds(dataBound);
785       yaxis.setLayoutBounds(dataBound);
786     
787       ColumnMetadata starttime_meta = maindatatable.getMetadata(START_FIELD_NAME);    
788       ColumnMetadata finishtime_meta = maindatatable.getMetadata(END_FIELD_NAME);    
789       ColumnMetadata ycoord_meta = maindatatable.getMetadata("ycoord");
790       long x_min = (long) ((Double)maindatatable.get(starttime_meta.getMinimumRow(), START_FIELD_NAME)).doubleValue();
791       long x_max = (long) ((Double)maindatatable.get(finishtime_meta.getMaximumRow(), END_FIELD_NAME)).doubleValue();
792       xaxis.setRangeModel(new NumberRangeModel(x_min,x_max,x_min,x_max));
793       float y_max = maindatatable.getFloat(ycoord_meta.getMaximumRow(),"ycoord");
794       yaxis.setRangeModel(new NumberRangeModel(0,y_max,0,y_max));
795       
796       // call custom action to plot actual swimlanes lines
797       CoordScaler cs = new CoordScaler();
798       cs.set_pixel_size(SIZE_X-BORDER[0]-BORDER[2], SIZE_Y-BORDER[1]-BORDER[3]);
799       cs.set_pixel_start(BORDER[0],BORDER[1]);
800       cs.set_value_ranges(x_min,0,x_max,y_max);
801       //SwimlanesStateAction swimlaneslines = new SwimlanesStateAction(maingroup, cs);
802       SwimlanesStateAction swimlaneslines = new SwimlanesStateAction(maingroup, cs);
803       
804       // add everything to the plot
805       draw.add(xaxis);
806       draw.add(yaxis);
807       draw.add(swimlaneslines);
808       
809       AxisLabelLayout xlabels = new AxisLabelLayout("xlab", xaxis, xlabBound);
810       this.viz.putAction("xlabels",xlabels);
811       AxisLabelLayout ylabels = new AxisLabelLayout("ylab", yaxis, ylabBound);
812       this.viz.putAction("ylabels",ylabels);
813     }
814     
815     // add axes names
816     {
817       SpecifiedLayout sl = new SpecifiedLayout(labelgroup, VisualItem.X, VisualItem.Y);
818       ActionList labeldraw = new ActionList();
819       labeldraw.add(sl);
820       this.viz.putAction(labelgroup, labeldraw);
821     }
822     
823     // add legend
824     if (plot_legend) {
825       ShapeAction legend_sa = new ShapeAction(legendshapegroup);
826       SpecifiedLayout legendlabels_sl = new SpecifiedLayout(legendgroup, VisualItem.X, VisualItem.Y);
827     
828       ActionList legenddraw = new ActionList();
829       legenddraw.add(legend_sa);
830       this.viz.putAction(legendshapegroup, legenddraw);
831       ActionList legendlabelsdraw = new ActionList();
832       legendlabelsdraw.add(legendlabels_sl);
833       this.viz.putAction(legendgroup,legendlabelsdraw);
834     }
835     
836     // draw everything else
837     this.viz.putAction("draw",draw);
838 
839     // finally draw
840     this.viz.run("draw");
841     this.viz.run("xlabels");
842     this.viz.run("ylabels");
843 
844   }
845   
846   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
847       "SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE", 
848       justification = "Dynamic based upon tables in the database")
849   public Table getData() {
850     // preliminary setup
851     OfflineTimeHandler time_offline;
852     TimeHandler time_online;
853     long start, end;
854     
855     if (offline_use) {
856       time_offline = new OfflineTimeHandler(param_map, this.timezone);
857       start = time_offline.getStartTime();
858       end = time_offline.getEndTime();
859     } else {
860       time_online = new TimeHandler(this.request, this.timezone);
861       start = time_online.getStartTime();
862       end = time_online.getEndTime();
863     }
864     
865     DatabaseWriter dbw = new DatabaseWriter(this.cluster);
866     String query;
867     
868     // setup query
869     if (this.shuffle_option != null && this.shuffle_option.equals("shuffles")) {
870       query = "select job_id,friendly_id,start_time,finish_time,start_time_millis,finish_time_millis,status,state_name,hostname from ["+this.table+"] where finish_time between '[start]' and '[end]'";
871     } else {
872       query = "select job_id,friendly_id,start_time,finish_time,start_time_millis,finish_time_millis,status,state_name,hostname from ["+this.table+"] where finish_time between '[start]' and '[end]' and not state_name like 'shuffle_local' and not state_name like 'shuffle_remote'";
873     }
874     if (this.jobname != null) {
875       query = query + " and job_id like '"+ this.jobname +"'";
876     }
877     Macro mp = new Macro(start,end,query);
878     query = mp.toString() + " order by start_time";
879     
880     Table rs_tab = null;    
881     DatabaseDataSource dds; 
882 
883     log.debug("Query: " + query);
884     // execute query
885     try {
886       dds = ConnectionFactory.getDatabaseConnection(dbw.getConnection());
887       rs_tab = dds.getData(query);
888     } catch (prefuse.data.io.DataIOException e) {
889       System.err.println("prefuse data IO error: " + e);
890       log.warn("prefuse data IO error: " + e);
891       return null;
892     } catch (SQLException e) {
893       System.err.println("Error in SQL: " + e + " in statement: " + query);
894       log.warn("Error in SQL: " + e + " in statement: " + query);
895       return null;
896     }
897     
898     HashMap<String, Integer> state_counts = new HashMap<String, Integer>();
899     for (int i = 0; i < rs_tab.getRowCount(); i++) {
900       String curr_state = rs_tab.getString(i, "state_name");
901       Integer cnt = state_counts.get(curr_state);
902       if (cnt == null) {
903         state_counts.put(curr_state, Integer.valueOf(1));
904       } else {
905         state_counts.remove(curr_state);
906         state_counts.put(curr_state, Integer.valueOf(cnt.intValue()+1));
907       }
908     }
909     
910     log.info("Search complete: #cols: " + rs_tab.getColumnCount() + "; #rows: " + rs_tab.getRowCount());
911     
912     return rs_tab;
913   }
914   
915 }