org.apache.hadoop.chukwa.extraction.demux

Class PostProcessorManager

java.lang.Object

org.apache.hadoop.chukwa.extraction.demux.PostProcessorManager

All Implemented Interfaces:

CHUKWA_CONSTANT

public class PostProcessorManager
extends Object
implements CHUKWA_CONSTANT

Field Summary

Fields

Modifier and Type	Field and Description
protected ChukwaConfiguration	conf
protected static HashMap<String,String>	dataSources
protected int	ERROR_SLEEP_TIME
static int	errorCount
protected org.apache.hadoop.fs.FileSystem	fs
protected boolean	isRunning

Fields inherited from interface org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT

ARCHIVES_IN_ERROR_DIR_NAME, ARCHIVES_MR_INPUT_DIR_NAME, ARCHIVES_MR_OUTPUT_DIR_NAME, ARCHIVES_PROCESSING_DIR_NAME, CHUKWA_ARCHIVE_DIR_FIELD, CHUKWA_ARCHIVE_MAX_ERROR_COUNT_FIELD, CHUKWA_DATA_SINK_DIR_FIELD, CHUKWA_DEMUX_MAX_ERROR_COUNT_FIELD, CHUKWA_DEMUX_REDUCER_COUNT_FIELD, CHUKWA_NAGIOS_HOST_FIELD, CHUKWA_NAGIOS_PORT_FIELD, CHUKWA_POST_PROCESS_DIR_FIELD, CHUKWA_POSTPROCESS_IN_ERROR_DIR_FIELD, CHUKWA_POSTPROCESS_MAX_ERROR_COUNT_FIELD, CHUKWA_REPORTING_HOST_FIELD, CHUKWA_ROOT_DIR_FIELD, CHUKWA_ROOT_REPOS_DIR_FIELD, DEFAULT_CHUKWA_DATASINK_DIR_NAME, DEFAULT_CHUKWA_LOGS_DIR_NAME, DEFAULT_CHUKWA_POSTPROCESS_DIR_NAME, DEFAULT_CHUKWA_ROOT_DIR_NAME, DEFAULT_DEMUX_IN_ERROR_DIR_NAME, DEFAULT_DEMUX_MR_INPUT_DIR_NAME, DEFAULT_DEMUX_MR_OUTPUT_DIR_NAME, DEFAULT_DEMUX_PROCESSING_DIR_NAME, DEFAULT_FINAL_ARCHIVES, DEFAULT_POSTPROCESS_IN_ERROR_DIR_NAME, DEFAULT_REPOS_DIR_NAME, HDFS_DEFAULT_NAME_FIELD, HIERARCHY_CONNECTOR, INCLUDE_KEY_IN_PARTITIONER, POST_DEMUX_DATA_LOADER, POST_DEMUX_SUCCESS_ACTION, WRITER_HDFS_FILESYSTEM_FIELD

Constructor Summary

Constructors

Constructor and Description
PostProcessorManager()
PostProcessorManager(ChukwaConfiguration conf)

Method Summary

Methods

Modifier and Type	Method and Description
boolean	deleteDirectory(String directory)
protected void	init()
static void	main(String[] args)
boolean	moveToInErrorDirectory(String sourceDirectory, String dirName, String inErrorDirectory)
org.apache.hadoop.fs.Path[]	movetoMainRepository(String sourceDirectory, String repoRootDirectory)
boolean	processDataLoaders(String directory)
boolean	processPostMoveTriggers(org.apache.hadoop.fs.Path[] files)
void	shutdown()
void	start()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

dataSources

protected static HashMap<String,String> dataSources

errorCount

public static int errorCount

ERROR_SLEEP_TIME

protected int ERROR_SLEEP_TIME

conf

protected ChukwaConfiguration conf

fs

protected org.apache.hadoop.fs.FileSystem fs

isRunning

protected volatile boolean isRunning

Constructor Detail

PostProcessorManager

public PostProcessorManager()
                     throws Exception

Throws:

Exception

PostProcessorManager

public PostProcessorManager(ChukwaConfiguration conf)
                     throws Exception

Throws:

Exception

Method Detail

init

protected void init()
             throws IOException,
                    URISyntaxException

Throws:

IOException

URISyntaxException

main

public static void main(String[] args)
                 throws Exception

Throws:

Exception

shutdown

public void shutdown()

start

public void start()

processDataLoaders

public boolean processDataLoaders(String directory)
                           throws IOException

Throws:

IOException

processPostMoveTriggers

public boolean processPostMoveTriggers(org.apache.hadoop.fs.Path[] files)
                                throws IOException

Throws:

IOException

movetoMainRepository

public org.apache.hadoop.fs.Path[] movetoMainRepository(String sourceDirectory,
                                               String repoRootDirectory)
                                                 throws Exception

Throws:

Exception

moveToInErrorDirectory

public boolean moveToInErrorDirectory(String sourceDirectory,
                             String dirName,
                             String inErrorDirectory)
                               throws Exception

Throws:

Exception

deleteDirectory

public boolean deleteDirectory(String directory)
                        throws IOException

Throws:

IOException